From 947e5010125f25a5f1ab8bf6f0494ea73fde14a6 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 12 Mar 2020 03:06:19 +0200 Subject: [PATCH 1/7] Add support for index aliases for SYS COLUMNS command --- .../xpack/ql/index/IndexResolver.java | 224 ++++++++++- .../xpack/ql/type/InvalidMappedField.java | 5 + .../sql/qa/single_node/SysColumnsIT.java | 13 + .../xpack/sql/qa/jdbc/SysColumnsTestCase.java | 358 ++++++++++++++++++ .../single-node-only/command-sys.csv-spec | 90 +++-- .../analysis/index/IndexResolverTests.java | 6 +- 6 files changed, 642 insertions(+), 54 deletions(-) create mode 100644 x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/SysColumnsIT.java create mode 100644 x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SysColumnsTestCase.java diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 5d003dd80bf08..106c72c2e1366 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.ql.index; import com.carrotsearch.hppc.cursors.ObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectObjectCursor; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.ActionListener; @@ -22,6 +23,7 @@ import org.elasticsearch.client.Client; import org.elasticsearch.cluster.metadata.AliasMetaData; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xpack.ql.QlIllegalArgumentException; @@ -41,9 +43,11 @@ import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -162,7 +166,6 @@ public boolean equals(Object obj) { private final String clusterName; private final DataTypeRegistry typeRegistry; - public IndexResolver(Client client, String clusterName, DataTypeRegistry typeRegistry) { this.client = client; this.clusterName = clusterName; @@ -294,7 +297,7 @@ public static IndexResolution mergedMappings(DataTypeRegistry typeRegistry, Stri } // merge all indices onto the same one - List indices = buildIndices(typeRegistry, indexNames, null, fieldCaps, i -> indexPattern, (n, types) -> { + List indices = buildIndices(typeRegistry, indexNames, null, fieldCaps, null, i -> indexPattern, (n, types) -> { StringBuilder errorMessage = new StringBuilder(); boolean hasUnmapped = types.containsKey(UNMAPPED); @@ -460,15 +463,31 @@ public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, bo FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); client.fieldCaps(fieldRequest, ActionListener.wrap( - response -> listener.onResponse( - separateMappings(typeRegistry, indexWildcard, javaRegex, response.getIndices(), response.get())), + response -> { + String[] indicesList = response.getIndices(); + GetAliasesRequest aliasRequest = new GetAliasesRequest() + .local(true) + .aliases("*") + .indices(indicesList) + .indicesOptions(IndicesOptions.lenientExpandOpen()); + client.admin().indices().getAliases(aliasRequest, wrap(aliases -> + listener.onResponse(separateMappings(typeRegistry, javaRegex, indicesList, response.get(), + aliases.getAliases())), + ex -> { + if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { + listener.onResponse(separateMappings(typeRegistry, javaRegex, indicesList, response.get(), null)); + } else { + listener.onFailure(ex); + } + })); + }, listener::onFailure)); } - public static List separateMappings(DataTypeRegistry typeRegistry, String indexPattern, String javaRegex, String[] indexNames, - Map> fieldCaps) { - return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, Function.identity(), (s, cap) -> null); + public static List separateMappings(DataTypeRegistry typeRegistry, String javaRegex, String[] indexNames, + Map> fieldCaps, ImmutableOpenMap> aliases) { + return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null); } private static class Fields { @@ -481,16 +500,26 @@ private static class Fields { * each field. */ private static List buildIndices(DataTypeRegistry typeRegistry, String[] indexNames, String javaRegex, - Map> fieldCaps, + Map> fieldCaps, ImmutableOpenMap> aliases, Function indexNameProcessor, BiFunction, InvalidMappedField> validityVerifier) { - if (indexNames == null || indexNames.length == 0) { + if ((indexNames == null || indexNames.length == 0) && (aliases == null || aliases.size() == 0)) { return emptyList(); } - final List resolvedIndices = asList(indexNames); - Map indices = new LinkedHashMap<>(resolvedIndices.size()); + Set resolvedAliases = new LinkedHashSet<>(); + if (aliases != null) { + Iterator>> iterator = aliases.iterator(); + while (iterator.hasNext()) { + for(AliasMetaData alias : iterator.next().value) { + resolvedAliases.add(alias.getAlias()); + } + } + } + + List resolvedIndices = new ArrayList<>(asList(indexNames)); + Map indices = new LinkedHashMap<>(resolvedIndices.size() + resolvedAliases.size()); Pattern pattern = javaRegex != null ? Pattern.compile(javaRegex) : null; // sort fields in reverse order to build the field hierarchy @@ -508,8 +537,10 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ continue; } - // apply verification + // apply verification for individual fields final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types); + // apply verification for fields belonging to index aliases + Map invalidFieldsForAliases = getInvalidFieldsForAliases(fieldName, types, aliases); // filter meta fields and unmapped FieldCapabilities unmapped = types.get(UNMAPPED); @@ -530,7 +561,7 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ List concreteIndices = null; if (capIndices != null) { if (unmappedIndices.isEmpty()) { - concreteIndices = asList(capIndices); + concreteIndices = new ArrayList<>(asList(capIndices)); } else { concreteIndices = new ArrayList<>(capIndices.length); for (String capIndex : capIndices) { @@ -544,30 +575,55 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ concreteIndices = resolvedIndices; } + // add to the list of concrete indices the aliases associated with these indices + Set uniqueAliases = new LinkedHashSet<>(); + if (aliases != null) { + for (String concreteIndex : concreteIndices) { + if (aliases.containsKey(concreteIndex)) { + List concreteIndexAliases = aliases.get(concreteIndex); + concreteIndexAliases.stream().forEach(e -> uniqueAliases.add(e.alias())); + } + } + concreteIndices.addAll(uniqueAliases); + } + // put the field in their respective mappings for (String index : concreteIndices) { - if (pattern == null || pattern.matcher(index).matches()) { - String indexName = indexNameProcessor.apply(index); + boolean isIndexAlias = uniqueAliases.contains(index); + if (pattern == null || pattern.matcher(index).matches() || isIndexAlias) { + String indexName = isIndexAlias ? index : indexNameProcessor.apply(index); Fields indexFields = indices.get(indexName); if (indexFields == null) { indexFields = new Fields(); indices.put(indexName, indexFields); } EsField field = indexFields.flattedMapping.get(fieldName); - if (field == null || (invalidField != null && (field instanceof InvalidMappedField) == false)) { + boolean createField = false; + if (isIndexAlias == false) { + if (field == null || (invalidField != null && (field instanceof InvalidMappedField) == false)) { + createField = true; + } + } + else { + if (field == null && invalidFieldsForAliases.get(index) == null) { + createField = true; + } + } + + if (createField) { int dot = fieldName.lastIndexOf('.'); /* * Looking up the "tree" at the parent fields here to see if the field is an alias. * When the upper elements of the "tree" have no elements in fieldcaps, then this is an alias field. But not * always: if there are two aliases - a.b.c.alias1 and a.b.c.alias2 - only one of them will be considered alias. */ - Holder isAlias = new Holder<>(false); + Holder isAliasFieldType = new Holder<>(false); if (dot >= 0) { String parentName = fieldName.substring(0, dot); if (indexFields.flattedMapping.get(parentName) == null) { // lack of parent implies the field is an alias if (fieldCaps.get(parentName) == null) { - isAlias.set(true); + isAliasFieldType.set(true); } } } @@ -575,7 +631,7 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ createField(typeRegistry, fieldName, fieldCaps, indexFields.hierarchicalMapping, indexFields.flattedMapping, s -> invalidField != null ? invalidField : createField(typeRegistry, s, typeCap.getType(), emptyMap(), typeCap.isAggregatable(), - isAlias.get())); + isAliasFieldType.get())); } } } @@ -590,4 +646,134 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ foundIndices.sort(Comparator.comparing(EsIndex::name)); return foundIndices; } + + + /* + * Checks if the field is valid (same type and same capabilities - searchable/aggregatable) across indices belonging to a list + * of aliases. + * A field can look like the example below (generated by field_caps API). + * "name": { + * "text": { + * "type": "text", + * "searchable": false, + * "aggregatable": false, + * "indices": [ + * "bar", + * "foo" + * ], + * "non_searchable_indices": [ + * "foo" + * ] + * }, + * "keyword": { + * "type": "keyword", + * "searchable": false, + * "aggregatable": true, + * "non_aggregatable_indices": [ + * "bar", "baz" + * ] + * } + * } + */ + private static Map getInvalidFieldsForAliases(String fieldName, Map types, + ImmutableOpenMap> aliases) { + if (aliases == null || aliases.isEmpty()) { + return emptyMap(); + } + Map invalidFields = new HashMap<>(); + Map> typesErrors = new HashMap<>(); + Map> aliasToIndices = new HashMap<>(); // map with aliases and their list of indices + + Iterator>> iter = aliases.iterator(); + while (iter.hasNext()) { + ObjectObjectCursor> index = iter.next(); + for (AliasMetaData aliasMetaData : index.value) { + String aliasName = aliasMetaData.alias(); + typesErrors.putIfAbsent(aliasName, new HashSet<>()); + aliasToIndices.putIfAbsent(aliasName, new HashSet()); + aliasToIndices.get(aliasName).add(index.key); + } + } + + // iterate over each type + for (Entry type : types.entrySet()) { + if (type.getKey() == UNMAPPED) { + continue; + } + String[] indices = type.getValue().indices(); + // if there is a list of indices where this field type is defined + if (indices != null) { + // Look at all these indices' aliases and add the type of the field to a list (Set) with unique elements. + // A valid mapping for a field in an index alias should contain only one type. If it doesn't this means that field + // is mapped as different types across the indices in this index alias. + for (String index : indices) { + List indexAliases = aliases.get(index); + if (indexAliases == null) { + continue; + } + for (AliasMetaData aliasMetaData : indexAliases) { + typesErrors.get(aliasMetaData.alias()).add(type.getKey()); + } + } + } + } + + for (Entry> entry : typesErrors.entrySet()) { + String aliasName = entry.getKey(); + // if, for the same index alias, there are multiple field types for this fieldName ie the index alias has indices where the same + // field name is of different types + if (entry.getValue().size() > 1) { + // consider the field as invalid, for the currently checked index alias + // the error message doesn't actually matter + invalidFields.put(aliasName, new InvalidMappedField(fieldName)); + } else { + // if the field type is the same across all this alias' indices, check the field's capabilities (searchable/aggregatable) + for (Entry type : types.entrySet()) { + if (type.getKey() == UNMAPPED) { + continue; + } + FieldCapabilities f = type.getValue(); + + // the existence of a list of non_aggregatable_indices is an indication that not all indices have the same capabilities + // but this list can contain indices belonging to other aliases, so we need to check only for this alias + if (f.nonAggregatableIndices() != null) { + Set aliasIndices = aliasToIndices.get(aliasName); + int nonAggregatableCount = 0; + // either all or none of the non-aggregatable indices belonging to a certain alias should be in this list + for (String nonAggIndex : f.nonAggregatableIndices()) { + if (aliasIndices.contains(nonAggIndex)) { + nonAggregatableCount++; + } + } + if (nonAggregatableCount > 0 && nonAggregatableCount != aliasIndices.size()) { + invalidFields.put(aliasName, new InvalidMappedField(fieldName)); + break; + } + } + + // perform the same check for non_searchable_indices list + if (f.nonSearchableIndices() != null) { + Set aliasIndices = aliasToIndices.get(aliasName); + int nonSearchableCount = 0; + // either all or none of the non-searchable indices belonging to a certain alias should be in this list + for (String nonSearchIndex : f.nonSearchableIndices()) { + if (aliasIndices.contains(nonSearchIndex)) { + nonSearchableCount++; + } + } + if (nonSearchableCount > 0 && nonSearchableCount != aliasIndices.size()) { + invalidFields.put(aliasName, new InvalidMappedField(fieldName)); + break; + } + } + } + } + } + + if (invalidFields.size() > 0) { + return invalidFields; + } + // everything checks + return emptyMap(); + } } \ No newline at end of file diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/InvalidMappedField.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/InvalidMappedField.java index c69407a25598b..29c633759ef26 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/InvalidMappedField.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/InvalidMappedField.java @@ -25,6 +25,11 @@ public InvalidMappedField(String name, String errorMessage) { this.errorMessage = errorMessage; } + public InvalidMappedField(String name) { + super(name, DataTypes.UNSUPPORTED, emptyMap(), false); + this.errorMessage = StringUtils.EMPTY; + } + public String errorMessage() { return errorMessage; } diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/SysColumnsIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/SysColumnsIT.java new file mode 100644 index 0000000000000..b2568d970bca1 --- /dev/null +++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/SysColumnsIT.java @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa.single_node; + +import org.elasticsearch.xpack.sql.qa.jdbc.SysColumnsTestCase; + +public class SysColumnsIT extends SysColumnsTestCase { + +} diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SysColumnsTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SysColumnsTestCase.java new file mode 100644 index 0000000000000..12a30834caf9a --- /dev/null +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/SysColumnsTestCase.java @@ -0,0 +1,358 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.sql.qa.jdbc; + +import org.elasticsearch.client.Request; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; + +import static org.elasticsearch.xpack.sql.qa.jdbc.ResultSetTestCase.updateMapping; + +public class SysColumnsTestCase extends JdbcIntegrationTestCase { + + public void testAliasWithIncompatibleTypes() throws Exception { + createIndexWithMapping("test1", builder -> { + builder.startObject("id").field("type", "keyword").endObject(); + builder.startObject("value").field("type", "double").endObject(); + }); + + createIndexWithMapping("test2", builder -> { + builder.startObject("id").field("type", "text").endObject(); + builder.startObject("value").field("type", "double").endObject(); + }); + + createAliases(builder -> { + builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); + }); + + assertResultsForQuery("SYS COLUMNS", new String[][] { + {"test1" ,"id" ,"KEYWORD"}, + {"test1" ,"value" ,"DOUBLE"}, + {"test2" ,"id" ,"TEXT"}, + {"test2" ,"value" ,"DOUBLE"}, + {"test_alias" ,"value" ,"DOUBLE"} + }); + } + + public void testAliasWithIncompatibleSearchableProperty() throws Exception { + createIndexWithMapping("test1", builder -> { + builder.startObject("id").field("type", "keyword").endObject(); + builder.startObject("value").field("type", "boolean").endObject(); + }); + + createIndexWithMapping("test2", builder -> { + builder.startObject("id").field("type", "keyword").field("index", false).endObject(); + builder.startObject("value").field("type", "boolean").endObject(); + }); + + createIndexWithMapping("test3", builder -> { + builder.startObject("id").field("type", "keyword").field("index", false).endObject(); + builder.startObject("value").field("type", "boolean").endObject(); + }); + + createIndexWithMapping("test4", builder -> { + builder.startObject("id").field("type", "keyword").field("index", false).endObject(); + builder.startObject("value").field("type", "boolean").endObject(); + }); + + createAliases(builder -> { + builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test3").field("alias", "test_alias2").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test4").field("alias", "test_alias2").endObject().endObject(); + }); + + assertResultsForQuery("SYS COLUMNS", new String[][] { + {"test1" ,"id" ,"KEYWORD"}, + {"test1" ,"value" ,"BOOLEAN"}, + {"test2" ,"id" ,"KEYWORD"}, + {"test2" ,"value" ,"BOOLEAN"}, + {"test3" ,"id" ,"KEYWORD"}, + {"test3" ,"value" ,"BOOLEAN"}, + {"test4" ,"id" ,"KEYWORD"}, + {"test4" ,"value" ,"BOOLEAN"}, + {"test_alias" ,"value" ,"BOOLEAN"}, + {"test_alias2" ,"id" ,"KEYWORD"}, + {"test_alias2" ,"value" ,"BOOLEAN"} + }); + } + + public void testAliasWithIncompatibleAggregatableProperty() throws Exception { + createIndexWithMapping("test1", builder -> { + builder.startObject("id").field("type", "text").field("fielddata", true).endObject(); + builder.startObject("value").field("type", "date").endObject(); + }); + + createIndexWithMapping("test2", builder -> { + builder.startObject("id").field("type", "text").endObject(); + builder.startObject("value").field("type", "date").endObject(); + }); + + createIndexWithMapping("test3", builder -> { + builder.startObject("id").field("type", "text").field("fielddata", true).endObject(); + builder.startObject("value").field("type", "date").endObject(); + }); + + createIndexWithMapping("test4", builder -> { + builder.startObject("id").field("type", "text").field("fielddata", true).endObject(); + builder.startObject("value").field("type", "date").endObject(); + }); + + createAliases(builder -> { + builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test3").field("alias", "test_alias2").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test4").field("alias", "test_alias2").endObject().endObject(); + }); + + assertResultsForQuery("SYS COLUMNS", new String[][] { + {"test1" ,"id" ,"TEXT"}, + {"test1" ,"value" ,"DATETIME"}, + {"test2" ,"id" ,"TEXT"}, + {"test2" ,"value" ,"DATETIME"}, + {"test3" ,"id" ,"TEXT"}, + {"test3" ,"value" ,"DATETIME"}, + {"test4" ,"id" ,"TEXT"}, + {"test4" ,"value" ,"DATETIME"}, + {"test_alias" ,"value" ,"DATETIME"}, + {"test_alias2","id" ,"TEXT"}, + {"test_alias2","value" ,"DATETIME"}, + }); + } + + public void testAliasWithIncompatibleTypesInSubfield() throws Exception { + createIndexWithMapping("test1", builder -> { + builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "keyword") + .endObject().endObject().endObject(); + builder.startObject("value").field("type", "date").startObject("fields").startObject("raw").field("type", "long") + .endObject().endObject().endObject(); + }); + + createIndexWithMapping("test2", builder -> { + builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer") + .endObject().endObject().endObject(); + builder.startObject("value").field("type", "date").startObject("fields").startObject("raw").field("type", "long") + .endObject().endObject().endObject(); + }); + + createAliases(builder -> { + builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); + }); + + assertResultsForQuery("SYS COLUMNS", new String[][] { + {"test1" ,"id" ,"TEXT"}, + {"test1" ,"id.raw" ,"KEYWORD"}, + {"test1" ,"value" ,"DATETIME"}, + {"test1" ,"value.raw","LONG"}, + {"test2" ,"id" ,"TEXT"}, + {"test2" ,"id.raw" ,"INTEGER"}, + {"test2" ,"value" ,"DATETIME"}, + {"test2" ,"value.raw","LONG"}, + {"test_alias" ,"id" ,"TEXT"}, + {"test_alias" ,"value" ,"DATETIME"}, + {"test_alias" ,"value.raw","LONG"}, + }); + } + + public void testAliasWithIncompatibleSearchablePropertyInSubfield() throws Exception { + createIndexWithMapping("test1", builder -> { + builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer") + .endObject().endObject().endObject(); + builder.startObject("value").field("type", "date").startObject("fields").startObject("raw").field("type", "long") + .endObject().endObject().endObject(); + }); + + createIndexWithMapping("test2", builder -> { + builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer") + .field("index", false).endObject().endObject().endObject(); + builder.startObject("value").field("type", "date").startObject("fields").startObject("raw").field("type", "long") + .endObject().endObject().endObject(); + }); + + createAliases(builder -> { + builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); + }); + + assertResultsForQuery("SYS COLUMNS", new String[][] { + {"test1" ,"id" ,"TEXT"}, + {"test1" ,"id.raw" ,"INTEGER"}, + {"test1" ,"value" ,"DATETIME"}, + {"test1" ,"value.raw","LONG"}, + {"test2" ,"id" ,"TEXT"}, + {"test2" ,"id.raw" ,"INTEGER"}, + {"test2" ,"value" ,"DATETIME"}, + {"test2" ,"value.raw","LONG"}, + {"test_alias" ,"id" ,"TEXT"}, + {"test_alias" ,"value" ,"DATETIME"}, + {"test_alias" ,"value.raw","LONG"}, + }); + } + + public void testAliasWithIncompatibleAggregatablePropertyInSubfield() throws Exception { + createIndexWithMapping("test1", builder -> { + builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer") + .field("doc_values", false).endObject().endObject().endObject(); + builder.startObject("value").field("type", "ip").startObject("fields").startObject("raw").field("type", "text") + .endObject().endObject().endObject(); + }); + + createIndexWithMapping("test2", builder -> { + builder.startObject("id").field("type", "text").startObject("fields").startObject("raw").field("type", "integer") + .endObject().endObject().endObject(); + builder.startObject("value").field("type", "ip").startObject("fields").startObject("raw").field("type", "text") + .endObject().endObject().endObject(); + }); + + createAliases(builder -> { + builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); + }); + + assertResultsForQuery("SYS COLUMNS", new String[][] { + {"test1" ,"id" ,"TEXT"}, + {"test1" ,"id.raw" ,"INTEGER"}, + {"test1" ,"value" ,"IP"}, + {"test1" ,"value.raw","TEXT"}, + {"test2" ,"id" ,"TEXT"}, + {"test2" ,"id.raw" ,"INTEGER"}, + {"test2" ,"value" ,"IP"}, + {"test2" ,"value.raw","TEXT"}, + {"test_alias" ,"id" ,"TEXT"}, + {"test_alias" ,"value" ,"IP"}, + {"test_alias" ,"value.raw","TEXT"}, + }); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/53445") + public void testAliasWithSubfieldsAndDifferentRootFields() throws Exception { + createIndexWithMapping("test1", builder -> { + builder.startObject("id").field("type", "keyword").endObject(); + builder.startObject("name").field("type", "text").startObject("fields").startObject("raw").field("type", "keyword") + .endObject().endObject().endObject(); + }); + + createIndexWithMapping("test2", builder -> { + builder.startObject("id").field("type", "keyword").endObject(); + builder.startObject("name").field("type", "keyword").field("index", false).startObject("fields").startObject("raw") + .field("type", "keyword").endObject().endObject().endObject(); + }); + + createAliases(builder -> { + builder.startObject().startObject("add").field("index", "test1").field("alias", "test_alias").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test2").field("alias", "test_alias").endObject().endObject(); + }); + + assertResultsForQuery("SYS COLUMNS", new String[][] { + {"test1" ,"id" ,"KEYWORD"}, + {"test1" ,"name" ,"TEXT"}, + {"test1" ,"name.raw","KEYWORD"}, + {"test2" ,"id" ,"KEYWORD"}, + {"test2" ,"name" ,"KEYWORD"}, + {"test2" ,"name.raw","KEYWORD"}, + {"test_alias" ,"id" ,"KEYWORD"} + }); + } + + public void testMultiIndicesMultiAlias() throws Exception { + createIndexWithMapping("test2", builder -> { + builder.startObject("id").field("type", "keyword").endObject(); + builder.startObject("name").field("type", "text").endObject(); + }); + createIndexWithMapping("test4", builder -> { + builder.startObject("id").field("type", "keyword").endObject(); + builder.startObject("name").field("type", "text").field("index", false).endObject(); + }); + createIndexWithMapping("test1", builder -> { + builder.startObject("id").field("type", "keyword").endObject(); + builder.startObject("name").field("type", "keyword").endObject(); + builder.startObject("number").field("type", "long").endObject(); + }); + createIndexWithMapping("test3", builder -> { + builder.startObject("id").field("type", "keyword").endObject(); + builder.startObject("name").field("type", "keyword").endObject(); + builder.startObject("number").field("type", "long").endObject(); + }); + + createAliases(builder -> { + builder.startObject().startObject("add").field("index", "test1").field("alias", "alias1").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test1").field("alias", "alias2").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test2").field("alias", "alias2").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test2").field("alias", "alias3").endObject().endObject(); + builder.startObject().startObject("add").field("index", "test4").field("alias", "alias3").endObject().endObject(); + }); + + assertResultsForQuery("SYS COLUMNS", new String[][] { + {"alias1","id" ,"KEYWORD"}, + {"alias1","name" ,"KEYWORD"}, + {"alias1","number","LONG"}, + {"alias2","id" ,"KEYWORD"}, + {"alias2","number","LONG"}, + {"alias3","id" ,"KEYWORD"}, + {"test1" ,"id" ,"KEYWORD"}, + {"test1" ,"name" ,"KEYWORD"}, + {"test1" ,"number","LONG"}, + {"test2" ,"id" ,"KEYWORD"}, + {"test2" ,"name" ,"TEXT"}, + {"test3" ,"id" ,"KEYWORD"}, + {"test3" ,"name" ,"KEYWORD"}, + {"test3" ,"number","LONG"}, + {"test4" ,"id" ,"KEYWORD"}, + {"test4" ,"name" ,"TEXT"} + }); + } + + private static void createIndexWithMapping(String indexName, CheckedConsumer mapping) throws Exception { + ResultSetTestCase.createIndex(indexName); + updateMapping(indexName, mapping); + } + + private void doWithQuery(String query, CheckedConsumer consumer) throws SQLException { + try (Connection connection = esJdbc()) { + try (PreparedStatement statement = connection.prepareStatement(query)) { + try (ResultSet results = statement.executeQuery()) { + consumer.accept(results); + } + } + } + } + + private static void createAliases(CheckedConsumer definitions) throws Exception { + Request request = new Request("POST", "/_aliases"); + XContentBuilder createAliases = JsonXContent.contentBuilder().startObject(); + createAliases.startArray("actions"); + { + definitions.accept(createAliases); + } + createAliases.endArray(); + createAliases.endObject(); + request.setJsonEntity(Strings.toString(createAliases)); + client().performRequest(request); + } + + private void assertResultsForQuery(String query, String[][] rows) throws Exception { + doWithQuery(query, (results) -> { + for (String[] row : rows) { + results.next(); + assertEquals(row[0], results.getString(3)); // table name + assertEquals(row[1], results.getString(4)); // column name + assertEquals(row[2], results.getString(6)); // type name + } + assertFalse(results.next()); + }); + } +} diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec index 34e093d1526ed..9e5f89c200399 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec @@ -88,36 +88,62 @@ sysColumnsAllTables SYS COLUMNS TABLE LIKE '%'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ----------------+---------------+---------------+--------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |logs |@timestamp |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_ip |12 |IP |0 |39 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO -integTest |null |logs |dest_ip |12 |IP |0 |39 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO -integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +---------------+---------------+---------------+------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +integTest |null |logs |@timestamp |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_ip |12 |IP |0 |39 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO +integTest |null |logs |dest_ip |12 |IP |0 |39 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO +integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO ; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index b53d1cbdf8133..3e87923cec768 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -326,7 +326,7 @@ public static IndexResolution merge(EsIndex... indices) { } public static List separate(EsIndex... indices) { - return separateMappings("*", null, Stream.of(indices).map(EsIndex::name).toArray(String[]::new), + return separateMappings(null, Stream.of(indices).map(EsIndex::name).toArray(String[]::new), fromMappings(indices)); } @@ -435,8 +435,8 @@ private static IndexResolution mergedMappings(String indexPattern, String[] inde return IndexResolver.mergedMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, indexNames, fieldCaps); } - private static List separateMappings(String indexPattern, String javaRegex, String[] indexNames, + private static List separateMappings(String javaRegex, String[] indexNames, Map> fieldCaps) { - return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, indexPattern, javaRegex, indexNames, fieldCaps); + return IndexResolver.separateMappings(SqlDataTypeRegistry.INSTANCE, javaRegex, indexNames, fieldCaps, null); } } From 3aa01f30490b36d998aa965c3c23acb32e8717da Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Fri, 13 Mar 2020 01:52:49 +0200 Subject: [PATCH 2/7] Update to test after branch update --- .../single-node-only/command-sys.csv-spec | 90 ++++++++++++------- 1 file changed, 58 insertions(+), 32 deletions(-) diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec index 711df01849103..62dd382aa8863 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec @@ -88,36 +88,62 @@ sysColumnsAllTables SYS COLUMNS TABLE LIKE '%'; TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ----------------+---------------+---------------+------------------+---------------+---------------+---------------+----------------+----------------+-----------------+-----------+---------------+---------------+---------------+------------------+-------------------+------------------+-------------+---------------+---------------+---------------+----------------+------------------+------------------ -integTest |null |logs |@timestamp |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO -integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO -integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +---------------+---------------+---------------+------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +integTest |null |logs |@timestamp |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO +integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO +integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO ; From 26bf119aaa30d170b7932116172326e7661b415c Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 16 Mar 2020 15:41:48 +0200 Subject: [PATCH 3/7] Address reviews --- .../xpack/ql/index/IndexResolver.java | 51 ++++++++++--------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 106c72c2e1366..0b3a8de7c0e82 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -17,6 +17,7 @@ import org.elasticsearch.action.admin.indices.get.GetIndexResponse; import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.IndicesOptions.Option; import org.elasticsearch.action.support.IndicesOptions.WildcardStates; @@ -461,30 +462,29 @@ private static FieldCapabilitiesRequest createFieldCapsRequest(String index, boo public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, boolean includeFrozen, ActionListener> listener) { FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen); - client.fieldCaps(fieldRequest, - ActionListener.wrap( - response -> { - String[] indicesList = response.getIndices(); - GetAliasesRequest aliasRequest = new GetAliasesRequest() - .local(true) - .aliases("*") - .indices(indicesList) - .indicesOptions(IndicesOptions.lenientExpandOpen()); - client.admin().indices().getAliases(aliasRequest, wrap(aliases -> - listener.onResponse(separateMappings(typeRegistry, javaRegex, indicesList, response.get(), - aliases.getAliases())), - ex -> { - if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { - listener.onResponse(separateMappings(typeRegistry, javaRegex, indicesList, response.get(), null)); - } else { - listener.onFailure(ex); - } - })); - }, - listener::onFailure)); + client.fieldCaps(fieldRequest, wrap(response -> { + client.admin().indices().getAliases(createGetAliasesRequest(response, includeFrozen), wrap(aliases -> + listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), aliases.getAliases())), + ex -> { + if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { + listener.onResponse(separateMappings(typeRegistry, javaRegex, response.getIndices(), response.get(), null)); + } else { + listener.onFailure(ex); + } + })); + }, + listener::onFailure)); } + private GetAliasesRequest createGetAliasesRequest(FieldCapabilitiesResponse response, boolean includeFrozen) { + return new GetAliasesRequest() + .local(true) + .aliases("*") + .indices(response.getIndices()) + .indicesOptions(includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS); + } + public static List separateMappings(DataTypeRegistry typeRegistry, String javaRegex, String[] indexNames, Map> fieldCaps, ImmutableOpenMap> aliases) { return buildIndices(typeRegistry, indexNames, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null); @@ -508,7 +508,7 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ return emptyList(); } - Set resolvedAliases = new LinkedHashSet<>(); + Set resolvedAliases = new HashSet<>(); if (aliases != null) { Iterator>> iterator = aliases.iterator(); while (iterator.hasNext()) { @@ -519,7 +519,8 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ } List resolvedIndices = new ArrayList<>(asList(indexNames)); - Map indices = new LinkedHashMap<>(resolvedIndices.size() + resolvedAliases.size()); + int mapSize = CollectionUtils.mapSize(resolvedIndices.size() + resolvedAliases.size()); + Map indices = new LinkedHashMap<>(mapSize); Pattern pattern = javaRegex != null ? Pattern.compile(javaRegex) : null; // sort fields in reverse order to build the field hierarchy @@ -681,7 +682,7 @@ private static Map getInvalidFieldsForAliases(String return emptyMap(); } Map invalidFields = new HashMap<>(); - Map> typesErrors = new HashMap<>(); + Map> typesErrors = new HashMap<>(); // map holding aliases and a list of unique field types accross its indices Map> aliasToIndices = new HashMap<>(); // map with aliases and their list of indices Iterator>> iter = aliases.iterator(); @@ -704,7 +705,7 @@ private static Map getInvalidFieldsForAliases(String // if there is a list of indices where this field type is defined if (indices != null) { // Look at all these indices' aliases and add the type of the field to a list (Set) with unique elements. - // A valid mapping for a field in an index alias should contain only one type. If it doesn't this means that field + // A valid mapping for a field in an index alias should contain only one type. If it doesn't, this means that field // is mapped as different types across the indices in this index alias. for (String index : indices) { List indexAliases = aliases.get(index); From 80f0b63a90d2db8032100b50659db27a9d787f05 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 16 Mar 2020 16:02:37 +0200 Subject: [PATCH 4/7] Fix one integration test after update branch --- .../single-node-only/command-sys.csv-spec | 121 +++++++++--------- 1 file changed, 62 insertions(+), 59 deletions(-) diff --git a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec index 2c739f2f073ff..4bc099d882142 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/single-node-only/command-sys.csv-spec @@ -90,63 +90,66 @@ integTest |null |test_alias |salary |4 |I sysColumnsAllTables SYS COLUMNS TABLE LIKE '%'; - TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i| BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s ----------------+---------------+---------------+------------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ -integTest |null |logs |@timestamp |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO -integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO -integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO -integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO -integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO -integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |16 |YES |null |null |null |null |NO |NO + TABLE_CAT:s | TABLE_SCHEM:s| TABLE_NAME:s | COLUMN_NAME:s | DATA_TYPE:i | TYPE_NAME:s | COLUMN_SIZE:i|BUFFER_LENGTH:i|DECIMAL_DIGITS:i|NUM_PREC_RADIX:i | NULLABLE:i| REMARKS:s | COLUMN_DEF:s |SQL_DATA_TYPE:i|SQL_DATETIME_SUB:i|CHAR_OCTET_LENGTH:i|ORDINAL_POSITION:i|IS_NULLABLE:s|SCOPE_CATALOG:s|SCOPE_SCHEMA:s|SCOPE_TABLE:s|SOURCE_DATA_TYPE:sh|IS_AUTOINCREMENT:s|IS_GENERATEDCOLUMN:s +-----------------+---------------+---------------+------------------+---------------+----------------+---------------+---------------+----------------+---------------+---------------+---------------+---------------+---------------+----------------+-----------------+----------------+---------------+---------------+---------------+---------------+----------------+----------------+------------------ +integTest |null |logs |@timestamp |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_in |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |2 |YES |null |null |null |null |NO |NO +integTest |null |logs |bytes_out |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |4 |YES |null |null |null |null |NO |NO +integTest |null |logs |client_port |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |5 |YES |null |null |null |null |NO |NO +integTest |null |logs |dest_ip |12 |IP |45 |45 |null |null |1 |null |null |12 |0 |null |6 |YES |null |null |null |null |NO |NO +integTest |null |logs |id |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |logs |status |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_gender |12 |CONSTANT_KEYWORD|32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |null_constant |12 |CONSTANT_KEYWORD|32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_gender |12 |CONSTANT_KEYWORD|32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |null_constant |12 |CONSTANT_KEYWORD|32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_alias_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |4 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |5 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |birth_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |1 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |emp_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |3 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra.info.gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |6 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_gender |12 |CONSTANT_KEYWORD|32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |7 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |extra_no |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |8 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |9 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |first_name.keyword|12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |10 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |gender |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |11 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |hire_date |93 |DATETIME |29 |8 |null |null |1 |null |null |9 |3 |null |12 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |languages |-6 |BYTE |5 |1 |null |10 |1 |null |null |-6 |0 |null |13 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name |12 |TEXT |2147483647 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |14 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |last_name.keyword |12 |KEYWORD |32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |15 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |null_constant |12 |CONSTANT_KEYWORD|32766 |2147483647 |null |null |1 |null |null |12 |0 |2147483647 |16 |YES |null |null |null |null |NO |NO +integTest |null |test_emp_copy |salary |4 |INTEGER |11 |4 |null |10 |1 |null |null |4 |0 |null |17 |YES |null |null |null |null |NO |NO ; From d34271a85cae13da46deff6a1d20075f26ef156e Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 16 Mar 2020 18:40:42 +0200 Subject: [PATCH 5/7] Address further review --- .../xpack/ql/index/IndexResolver.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index ee4189d16df39..ab58810da8ebd 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -705,7 +705,6 @@ private static Map getInvalidFieldsForAliases(String ObjectObjectCursor> index = iter.next(); for (AliasMetaData aliasMetaData : index.value) { String aliasName = aliasMetaData.alias(); - typesErrors.putIfAbsent(aliasName, new HashSet<>()); aliasToIndices.putIfAbsent(aliasName, new HashSet()); aliasToIndices.get(aliasName).add(index.key); } @@ -713,7 +712,8 @@ private static Map getInvalidFieldsForAliases(String // iterate over each type for (Entry type : types.entrySet()) { - if (type.getKey() == UNMAPPED) { + String esFieldType = type.getKey(); + if (esFieldType == UNMAPPED) { continue; } String[] indices = type.getValue().indices(); @@ -728,17 +728,24 @@ private static Map getInvalidFieldsForAliases(String continue; } for (AliasMetaData aliasMetaData : indexAliases) { - typesErrors.get(aliasMetaData.alias()).add(type.getKey()); + String aliasName = aliasMetaData.alias(); + if (typesErrors.containsKey(aliasName)) { + typesErrors.get(aliasName).add(esFieldType); + } else { + Set fieldTypes = new HashSet<>(); + fieldTypes.add(esFieldType); + typesErrors.put(aliasName, fieldTypes); + } } } } } - for (Entry> entry : typesErrors.entrySet()) { - String aliasName = entry.getKey(); + for (String aliasName : aliasToIndices.keySet()) { // if, for the same index alias, there are multiple field types for this fieldName ie the index alias has indices where the same // field name is of different types - if (entry.getValue().size() > 1) { + Set esFieldTypes = typesErrors.get(aliasName); + if (esFieldTypes != null && esFieldTypes.size() > 1) { // consider the field as invalid, for the currently checked index alias // the error message doesn't actually matter invalidFields.put(aliasName, new InvalidMappedField(fieldName)); From 9ed8fa2667d18e3be724fa8c3857ea0aeb87f864 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 17 Mar 2020 07:45:38 +0200 Subject: [PATCH 6/7] Address more reviews --- .../java/org/elasticsearch/xpack/ql/index/IndexResolver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index ab58810da8ebd..6c688b0a4bdb1 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -519,7 +519,7 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ Function indexNameProcessor, BiFunction, InvalidMappedField> validityVerifier) { - if ((indexNames == null || indexNames.length == 0) && (aliases == null || aliases.size() == 0)) { + if ((indexNames == null || indexNames.length == 0) && (aliases == null || aliases.isEmpty())) { return emptyList(); } @@ -697,7 +697,7 @@ private static Map getInvalidFieldsForAliases(String return emptyMap(); } Map invalidFields = new HashMap<>(); - Map> typesErrors = new HashMap<>(); // map holding aliases and a list of unique field types accross its indices + Map> typesErrors = new HashMap<>(); // map holding aliases and a list of unique field types across its indices Map> aliasToIndices = new HashMap<>(); // map with aliases and their list of indices Iterator>> iter = aliases.iterator(); From 079be6ba8fa806f4851be9af529bc72ede5e8d0e Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 17 Mar 2020 10:43:37 +0200 Subject: [PATCH 7/7] Small updates --- .../java/org/elasticsearch/xpack/ql/index/IndexResolver.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 6c688b0a4bdb1..cfc125d492519 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -527,7 +527,7 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ if (aliases != null) { Iterator>> iterator = aliases.iterator(); while (iterator.hasNext()) { - for(AliasMetaData alias : iterator.next().value) { + for (AliasMetaData alias : iterator.next().value) { resolvedAliases.add(alias.getAlias()); } } @@ -705,7 +705,7 @@ private static Map getInvalidFieldsForAliases(String ObjectObjectCursor> index = iter.next(); for (AliasMetaData aliasMetaData : index.value) { String aliasName = aliasMetaData.alias(); - aliasToIndices.putIfAbsent(aliasName, new HashSet()); + aliasToIndices.putIfAbsent(aliasName, new HashSet<>()); aliasToIndices.get(aliasName).add(index.key); } }