QL: add unsigned_long type support

docs/changelog/65145.yaml

@@ -0,0 +1,6 @@
+pr: 65145
+summary: Add `unsigned_long` type support
+area: Query Languages
+type: enhancement
+issues:
+ - 63312

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlSession.java

@@ -7,6 +7,7 @@
 
 package org.elasticsearch.xpack.eql.session;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.internal.Client;
 import org.elasticsearch.client.internal.ParentTaskAssigningClient;
@@ -120,6 +121,7 @@ private <T> void preAnalyze(LogicalPlan parsed, ActionListener<LogicalPlan> list
             indexWildcard,
             configuration.indicesOptions(),
             configuration.runtimeMappings(),
+            Version.CURRENT,
             map(listener, r -> preAnalyzer.preAnalyze(parsed, r))
         );
     }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/execution/search/QlSourceBuilder.java

@@ -22,7 +22,6 @@
  * the resulting ES document as a field.
  */
 public class QlSourceBuilder {
-    public static final Version SWITCH_TO_FIELDS_API_VERSION = Version.V_7_10_0;
     public static final Version INTRODUCING_MISSING_ORDER_IN_COMPOSITE_AGGS_VERSION = Version.V_7_16_0;
     // The LinkedHashMaps preserve the order of the fields in the response
     private final Set<FieldAndFormat> fetchFields = new LinkedHashSet<>();

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/function/scalar/ScalarFunction.java

@@ -158,7 +158,7 @@ protected ScriptTemplate scriptWithField(FieldAttribute field) {
         return field.dataType() != UNSIGNED_LONG
             ? new ScriptTemplate(processScript(Scripts.DOC_VALUE), params, dataType())
             : new ScriptTemplate(
-                processScript(formatTemplate(format("{ql}.", "nullSafeCastToUnsignedLong({})", Scripts.DOC_VALUE))),
+                processScript(format("{ql}.", "nullSafeCastToUnsignedLong({})", Scripts.DOC_VALUE)),
                 params,
                 UNSIGNED_LONG
             );

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java

@@ -9,6 +9,7 @@
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
 
 import org.elasticsearch.ElasticsearchSecurityException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
 import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
@@ -68,10 +69,12 @@
 import static org.elasticsearch.common.Strings.hasText;
 import static org.elasticsearch.common.regex.Regex.simpleMatch;
 import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName;
+import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.supportsUnsignedLong;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED;
 import static org.elasticsearch.xpack.ql.util.StringUtils.qualifyAndJoinIndices;
 import static org.elasticsearch.xpack.ql.util.StringUtils.splitQualifiedIndex;
@@ -366,12 +369,16 @@ public void resolveAsMergedMapping(
         String indexWildcard,
         IndicesOptions indicesOptions,
         Map<String, Object> runtimeMappings,
+        Version version,
         ActionListener<IndexResolution> listener
     ) {
         FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, indicesOptions, runtimeMappings);
         client.fieldCaps(
             fieldRequest,
-            ActionListener.wrap(response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), listener::onFailure)
+            ActionListener.wrap(
+                response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response, version)),
+                listener::onFailure
+            )
         );
     }
 
@@ -382,27 +389,32 @@ public void resolveAsMergedMapping(
         String indexWildcard,
         boolean includeFrozen,
         Map<String, Object> runtimeMappings,
+        Version version,
         ActionListener<IndexResolution> listener
     ) {
         FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, runtimeMappings);
         client.fieldCaps(
             fieldRequest,
-            ActionListener.wrap(response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response)), listener::onFailure)
+            ActionListener.wrap(
+                response -> listener.onResponse(mergedMappings(typeRegistry, indexWildcard, response, version)),
+                listener::onFailure
+            )
         );
     }
 
     public static IndexResolution mergedMappings(
         DataTypeRegistry typeRegistry,
         String indexPattern,
-        FieldCapabilitiesResponse fieldCapsResponse
+        FieldCapabilitiesResponse fieldCapsResponse,
+        Version version
     ) {
 
         if (fieldCapsResponse.getIndices().length == 0) {
             return IndexResolution.notFound(indexPattern);
         }
 
         // merge all indices onto the same one
-        List<EsIndex> indices = buildIndices(typeRegistry, null, fieldCapsResponse, null, i -> indexPattern, (n, types) -> {
+        List<EsIndex> indices = buildIndices(typeRegistry, null, fieldCapsResponse, null, i -> indexPattern, version, (n, types) -> {
             StringBuilder errorMessage = new StringBuilder();
 
             boolean hasUnmapped = types.containsKey(UNMAPPED);
@@ -473,7 +485,8 @@ private static EsField createField(
         Map<String, Map<String, FieldCapabilities>> globalCaps,
         Map<String, EsField> hierarchicalMapping,
         Map<String, EsField> flattedMapping,
-        Function<String, EsField> field
+        Function<String, EsField> field,
+        Version version
     ) {
 
         Map<String, EsField> parentProps = hierarchicalMapping;
@@ -493,18 +506,26 @@ private static EsField createField(
                 // lack of parent implies the field is an alias
                 if (map == null) {
                     // as such, create the field manually, marking the field to also be an alias
-                    fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true);
+                    fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), new TreeMap<>(), false, true, version);
                 } else {
                     Iterator<FieldCapabilities> iterator = map.values().iterator();
                     FieldCapabilities parentCap = iterator.next();
                     if (iterator.hasNext() && UNMAPPED.equals(parentCap.getType())) {
                         parentCap = iterator.next();
                     }
                     final FieldCapabilities parentC = parentCap;
-                    fieldFunction = s -> createField(typeRegistry, s, parentC.getType(), new TreeMap<>(), parentC.isAggregatable(), false);
+                    fieldFunction = s -> createField(
+                        typeRegistry,
+                        s,
+                        parentC.getType(),
+                        new TreeMap<>(),
+                        parentC.isAggregatable(),
+                        false,
+                        version
+                    );
                 }
 
-                parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction);
+                parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction, version);
             }
             parentProps = parent.getProperties();
         }
@@ -537,7 +558,8 @@ private static EsField createField(
         String typeName,
         Map<String, EsField> props,
         boolean isAggregateable,
-        boolean isAlias
+        boolean isAlias,
+        Version version
     ) {
         DataType esType = typeRegistry.fromEs(typeName);
 
@@ -553,7 +575,7 @@ private static EsField createField(
         if (esType == DATETIME) {
             return DateEsField.dateEsField(fieldName, props, isAggregateable);
         }
-        if (esType == UNSUPPORTED) {
+        if (esType == UNSUPPORTED || (esType == UNSIGNED_LONG && supportsUnsignedLong(version) == false)) {
             return new UnsupportedEsField(fieldName, typeName, null, props);
         }
 
@@ -591,6 +613,7 @@ public void resolveAsSeparateMappings(
         String javaRegex,
         boolean includeFrozen,
         Map<String, Object> runtimeMappings,
+        Version version,
         ActionListener<List<EsIndex>> listener
     ) {
         FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, includeFrozen, runtimeMappings);
@@ -600,10 +623,12 @@ public void resolveAsSeparateMappings(
                 .getAliases(
                     createGetAliasesRequest(response, includeFrozen),
                     wrap(
-                        aliases -> { listener.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases())); },
+                        aliases -> {
+                            listener.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases(), version));
+                        },
                         ex -> {
                             if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) {
-                                listener.onResponse(separateMappings(typeRegistry, javaRegex, response, null));
+                                listener.onResponse(separateMappings(typeRegistry, javaRegex, response, null, version));
                             } else {
                                 listener.onFailure(ex);
                             }
@@ -625,9 +650,10 @@ public static List<EsIndex> separateMappings(
         DataTypeRegistry typeRegistry,
         String javaRegex,
         FieldCapabilitiesResponse fieldCaps,
-        ImmutableOpenMap<String, List<AliasMetadata>> aliases
+        ImmutableOpenMap<String, List<AliasMetadata>> aliases,
+        Version version
     ) {
-        return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null);
+        return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), version, (s, cap) -> null);
     }
 
     private static class Fields {
@@ -645,6 +671,7 @@ private static List<EsIndex> buildIndices(
         FieldCapabilitiesResponse fieldCapsResponse,
         ImmutableOpenMap<String, List<AliasMetadata>> aliases,
         Function<String, String> indexNameProcessor,
+        Version version,
         BiFunction<String, Map<String, FieldCapabilities>, InvalidMappedField> validityVerifier
     ) {
 
@@ -780,8 +807,10 @@ private static List<EsIndex> buildIndices(
                                         typeCap.getType(),
                                         emptyMap(),
                                         typeCap.isAggregatable(),
-                                        isAliasFieldType.get()
-                                    )
+                                        isAliasFieldType.get(),
+                                        version
+                                    ),
+                                version
                             );
                         }
                     }

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/VersionCompatibilityChecks.java → x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/VersionCompatibilityChecks.java

@@ -5,25 +5,22 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.sql.session;
+package org.elasticsearch.xpack.ql.index;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.xpack.ql.type.DataType;
-import org.elasticsearch.xpack.sql.proto.SqlVersion;
 
 import static org.elasticsearch.Version.V_8_1_0;
 import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
 
 public final class VersionCompatibilityChecks {
 
-    public static final SqlVersion INTRODUCING_UNSIGNED_LONG = SqlVersion.fromId(V_8_1_0.id);
+    public static final Version INTRODUCING_UNSIGNED_LONG = V_8_1_0;
 
     private VersionCompatibilityChecks() {}
 
-    /**
-     * Is the provided {@code dataType} being supported in the provided {@code version}?
-     */
-    public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion version) {
+    public static boolean isTypeSupportedInVersion(DataType dataType, Version version) {
         if (dataType == UNSIGNED_LONG) {
             return supportsUnsignedLong(version);
         }
@@ -33,11 +30,11 @@ public static boolean isTypeSupportedInVersion(DataType dataType, SqlVersion ver
     /**
      * Does the provided {@code version} support the unsigned_long type (PR#60050)?
      */
-    public static boolean supportsUnsignedLong(SqlVersion version) {
+    public static boolean supportsUnsignedLong(Version version) {
         return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0;
     }
 
-    public static @Nullable SqlVersion versionIntroducingType(DataType dataType) {
+    public static @Nullable Version versionIntroducingType(DataType dataType) {
         if (dataType == UNSIGNED_LONG) {
             return INTRODUCING_UNSIGNED_LONG;
         }

x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/Types.java

@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.ql.type;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.core.Booleans;
 
@@ -15,32 +16,34 @@
 import java.util.Map.Entry;
 
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.ql.index.VersionCompatibilityChecks.supportsUnsignedLong;
 import static org.elasticsearch.xpack.ql.type.DataTypes.DATETIME;
 import static org.elasticsearch.xpack.ql.type.DataTypes.KEYWORD;
 import static org.elasticsearch.xpack.ql.type.DataTypes.NESTED;
 import static org.elasticsearch.xpack.ql.type.DataTypes.OBJECT;
 import static org.elasticsearch.xpack.ql.type.DataTypes.TEXT;
+import static org.elasticsearch.xpack.ql.type.DataTypes.UNSIGNED_LONG;
 import static org.elasticsearch.xpack.ql.type.DataTypes.UNSUPPORTED;
 
 public abstract class Types {
 
     @SuppressWarnings("unchecked")
-    public static Map<String, EsField> fromEs(DataTypeRegistry typeRegistry, Map<String, Object> asMap) {
+    public static Map<String, EsField> fromEs(DataTypeRegistry typeRegistry, Map<String, Object> asMap, Version version) {
         Map<String, Object> props = null;
         if (asMap != null && asMap.isEmpty() == false) {
             props = (Map<String, Object>) asMap.get("properties");
         }
-        return props == null || props.isEmpty() ? emptyMap() : startWalking(typeRegistry, props);
+        return props == null || props.isEmpty() ? emptyMap() : startWalking(typeRegistry, props, version);
     }
 
-    private static Map<String, EsField> startWalking(DataTypeRegistry typeRegistry, Map<String, Object> mapping) {
+    private static Map<String, EsField> startWalking(DataTypeRegistry typeRegistry, Map<String, Object> mapping, Version version) {
         Map<String, EsField> types = new LinkedHashMap<>();
 
         if (mapping == null) {
             return emptyMap();
         }
         for (Entry<String, Object> entry : mapping.entrySet()) {
-            walkMapping(typeRegistry, entry.getKey(), entry.getValue(), types);
+            walkMapping(typeRegistry, entry.getKey(), entry.getValue(), types, version);
         }
 
         return types;
@@ -65,7 +68,13 @@ private static DataType getType(DataTypeRegistry typeRegistry, Map<String, Objec
     }
 
     @SuppressWarnings("unchecked")
-    private static void walkMapping(DataTypeRegistry typeRegistry, String name, Object value, Map<String, EsField> mapping) {
+    private static void walkMapping(
+        DataTypeRegistry typeRegistry,
+        String name,
+        Object value,
+        Map<String, EsField> mapping,
+        Version version
+    ) {
         // object type - only root or nested docs supported
         if (value instanceof Map) {
             Map<String, Object> content = (Map<String, Object>) value;
@@ -74,17 +83,17 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje
             DataType esDataType = getType(typeRegistry, content);
             final Map<String, EsField> properties;
             if (esDataType == OBJECT || esDataType == NESTED) {
-                properties = fromEs(typeRegistry, content);
+                properties = fromEs(typeRegistry, content, version);
             } else if (content.containsKey("fields")) {
                 // Check for multifields
                 Object fields = content.get("fields");
                 if (fields instanceof Map) {
-                    properties = startWalking(typeRegistry, (Map<String, Object>) fields);
+                    properties = startWalking(typeRegistry, (Map<String, Object>) fields, version);
                 } else {
                     properties = Collections.emptyMap();
                 }
             } else {
-                properties = fromEs(typeRegistry, content);
+                properties = fromEs(typeRegistry, content, version);
             }
             boolean docValues = boolSetting(content.get("doc_values"), esDataType.hasDocValues());
             final EsField field;
@@ -96,7 +105,7 @@ private static void walkMapping(DataTypeRegistry typeRegistry, String name, Obje
                 field = new KeywordEsField(name, properties, docValues, length, normalized);
             } else if (esDataType == DATETIME) {
                 field = DateEsField.dateEsField(name, properties, docValues);
-            } else if (esDataType == UNSUPPORTED) {
+            } else if (esDataType == UNSUPPORTED || (esDataType == UNSIGNED_LONG && supportsUnsignedLong(version) == false)) {
                 String type = content.get("type").toString();
                 field = new UnsupportedEsField(name, type, null, properties);
                 propagateUnsupportedType(name, type, properties);