[client] Add client metadata header on RestClient requests (#68353)

Adds a X-Elastic-Client-Meta header to http requests sent by RestClient. This
header contains information about the runtime environment that is meant to
allow analyzing usage context by collecting this information on the receiving
side of requests, like a proxy server in front of ES.

Using a custom header allows client applications to change the User-Agent
header for their own purpose without losing this information.

Backport of #66303

Author: swallez

client/rest/build.gradle

@@ -43,6 +43,10 @@ dependencies {
   testImplementation "org.elasticsearch:mocksocket:${versions.mocksocket}"
 }
 
+tasks.named("processResources").configure {
+  expand versions: versions
+}
+
 tasks.withType(CheckForbiddenApis).configureEach {
   //client does not depend on server, so only jdk and http signatures should be checked
   replaceSignatureFiles('jdk-signatures', 'http-signatures')

client/rest/src/main/java/org/elasticsearch/client/LanguageRuntimeVersions.java

@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+
+// Copied verbatim from https://github.com/elastic/jvm-languages-sniffer
+
+class LanguageRuntimeVersions {
+
+    /**
+     * Returns runtime information by looking up classes identifying non-Java JVM
+     * languages and appending a key with their name and their major.minor version, if available
+     */
+    public static String getRuntimeMetadata() {
+        StringBuilder s = new StringBuilder();
+        String version;
+
+        version= kotlinVersion();
+        if (version != null) {
+            s.append(",kt=").append(version);
+        }
+
+        version = scalaVersion();
+        if (version != null) {
+            s.append(",sc=").append(version);
+        }
+
+        version = clojureVersion();
+        if (version != null) {
+            s.append(",clj=").append(version);
+        }
+
+        version = groovyVersion();
+        if (version != null) {
+            s.append(",gy=").append(version);
+        }
+
+        version = jRubyVersion();
+        if (version != null) {
+            s.append(",jrb=").append(version);
+        }
+
+        return s.toString();
+    }
+
+    public static String kotlinVersion() {
+        //KotlinVersion.CURRENT.toString()
+        return keepMajorMinor(getStaticField("kotlin.KotlinVersion", "CURRENT"));
+    }
+
+    public static String scalaVersion() {
+        // scala.util.Properties.versionNumberString()
+        return keepMajorMinor(callStaticMethod("scala.util.Properties", "versionNumberString"));
+    }
+
+    public static String clojureVersion() {
+        // (clojure-version) which translates to
+        // clojure.core$clojure_version.invokeStatic()
+        return keepMajorMinor(callStaticMethod("clojure.core$clojure_version", "invokeStatic"));
+    }
+
+    public static String groovyVersion() {
+        // groovy.lang.GroovySystem.getVersion()
+        // There's also getShortVersion(), but only since Groovy 3.0.1
+        return keepMajorMinor(callStaticMethod("groovy.lang.GroovySystem", "getVersion"));
+    }
+
+    public static String jRubyVersion() {
+        // org.jruby.runtime.Constants.VERSION
+        return keepMajorMinor(getStaticField("org.jruby.runtime.Constants", "VERSION"));
+    }
+
+    private static String getStaticField(String className, String fieldName) {
+        Class<?> clazz;
+        try {
+            clazz = Class.forName(className);
+        } catch (ClassNotFoundException e) {
+            return null;
+        }
+
+        try {
+            Field field = clazz.getField(fieldName);
+            return field.get(null).toString();
+        } catch (Exception e) {
+            return ""; // can't get version information
+        }
+    }
+
+    private static String callStaticMethod(String className, String methodName) {
+        Class<?> clazz;
+        try {
+            clazz = Class.forName(className);
+        } catch (ClassNotFoundException e) {
+            return null;
+        }
+
+        try {
+            Method m = clazz.getMethod(methodName);
+            return m.invoke(null).toString();
+        } catch (Exception e) {
+            return ""; // can't get version information
+        }
+    }
+
+    static String keepMajorMinor(String version) {
+        if (version == null) {
+            return null;
+        }
+
+        int firstDot = version.indexOf('.');
+        int secondDot = version.indexOf('.', firstDot + 1);
+        if (secondDot < 0) {
+            return version;
+        } else {
+            return version.substring(0, secondDot);
+        }
+    }
+}

client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java

@@ -20,19 +20,26 @@
 package org.elasticsearch.client;
 
 import org.apache.http.Header;
+import org.apache.http.HttpRequest;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
 import org.apache.http.impl.nio.client.HttpAsyncClientBuilder;
 import org.apache.http.nio.conn.SchemeIOSessionStrategy;
+import org.apache.http.protocol.HttpContext;
+import org.apache.http.util.VersionInfo;
 
 import javax.net.ssl.SSLContext;
+import java.io.IOException;
+import java.io.InputStream;
 import java.security.AccessController;
 import java.security.NoSuchAlgorithmException;
 import java.security.PrivilegedAction;
 import java.util.List;
+import java.util.Locale;
 import java.util.Objects;
+import java.util.Properties;
 
 /**
  * Helps creating a new {@link RestClient}. Allows to set the most common http client configuration options when internally
@@ -45,6 +52,11 @@ public final class RestClientBuilder {
     public static final int DEFAULT_MAX_CONN_PER_ROUTE = 10;
     public static final int DEFAULT_MAX_CONN_TOTAL = 30;
 
+    static final String VERSION;
+    static final String META_HEADER_NAME = "X-Elastic-Client-Meta";
+    private static final String META_HEADER_VALUE;
+    private static final String USER_AGENT_HEADER_VALUE;
+
     private static final Header[] EMPTY_HEADERS = new Header[0];
 
     private final List<Node> nodes;
@@ -56,6 +68,48 @@ public final class RestClientBuilder {
     private NodeSelector nodeSelector = NodeSelector.ANY;
     private boolean strictDeprecationMode = false;
     private boolean compressionEnabled = false;
+    private boolean metaHeaderEnabled = true;
+
+    static {
+
+        // Never fail on unknown version, even if an environment messed up their classpath enough that we can't find it.
+        // Better have incomplete telemetry than crashing user applications.
+        String version = null;
+        try (InputStream is = RestClient.class.getResourceAsStream("version.properties")) {
+            if (is != null) {
+                Properties versions = new Properties();
+                versions.load(is);
+                version = versions.getProperty("elasticsearch-client");
+            }
+        } catch (IOException e) {
+            // Keep version unknown
+        }
+
+        if (version == null) {
+            version = ""; // unknown values are reported as empty strings in X-Elastic-Client-Meta
+        }
+
+        VERSION = version;
+
+        USER_AGENT_HEADER_VALUE = String.format(Locale.ROOT, "elasticsearch-java/%s (Java/%s)",
+            VERSION.isEmpty() ? "Unknown" : VERSION, System.getProperty("java.version"));
+
+        VersionInfo httpClientVersion = null;
+        try {
+            httpClientVersion = AccessController.doPrivileged((PrivilegedAction<VersionInfo>)() ->
+                VersionInfo.loadVersionInfo("org.apache.http.nio.client", HttpAsyncClientBuilder.class.getClassLoader())
+            );
+        } catch (Exception e) {
+            // Keep unknown
+        }
+
+        // service, language, transport, followed by additional information
+        META_HEADER_VALUE = "es=" + VERSION +
+            ",jv=" + System.getProperty("java.specification.version") +
+            ",t=" + VERSION +
+            ",hc=" + (httpClientVersion == null ? "" : httpClientVersion.getRelease()) +
+            LanguageRuntimeVersions.getRuntimeMetadata();
+    }
 
     /**
      * Creates a new builder instance and sets the hosts that the client will send requests to.
@@ -191,6 +245,17 @@ public RestClientBuilder setCompressionEnabled(boolean compressionEnabled) {
         return this;
     }
 
+    /**
+     * Whether to send a {@code X-Elastic-Client-Meta} header that describes the runtime environment. It contains
+     * information that is similar to what could be found in {@code User-Agent}. Using a separate header allows
+     * applications to use {@code User-Agent} for their own needs, e.g. to identify application version or other
+     * environment information. Defaults to {@code true}.
+     */
+    public RestClientBuilder setMetaHeaderEnabled(boolean metadataEnabled) {
+        this.metaHeaderEnabled = metadataEnabled;
+        return this;
+    }
+
     /**
      * Creates a new {@link RestClient} based on the provided configuration.
      */
@@ -220,11 +285,20 @@ private CloseableHttpAsyncClient createHttpClient() {
                 //default settings for connection pooling may be too constraining
                 .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE).setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL)
                 .setSSLContext(SSLContext.getDefault())
+                .setUserAgent(USER_AGENT_HEADER_VALUE)
                 .setTargetAuthenticationStrategy(new PersistentCredentialsAuthenticationStrategy());
             if (httpClientConfigCallback != null) {
                 httpClientBuilder = httpClientConfigCallback.customizeHttpClient(httpClientBuilder);
             }
 
+            // Always add metadata header last so that it's not overwritten
+            httpClientBuilder.addInterceptorLast((HttpRequest request, HttpContext context) -> {
+                if (metaHeaderEnabled) {
+                    request.setHeader(META_HEADER_NAME, META_HEADER_VALUE);
+                } else {
+                    request.removeHeaders(META_HEADER_NAME);
+                }
+            });
             final HttpAsyncClientBuilder finalBuilder = httpClientBuilder;
             return AccessController.doPrivileged((PrivilegedAction<CloseableHttpAsyncClient>) finalBuilder::build);
         } catch (NoSuchAlgorithmException e) {

client/rest/src/main/resources/org/elasticsearch/client/version.properties

@@ -0,0 +1,20 @@
+#
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+elasticsearch-client=${versions.elasticsearch}