Merge branch 'master' into ccr

* master:
  992c788 Uncouple persistent task state and status (#31031)
  8c6ee7d Describe how to add a plugin in Dockerfile (#31340)
  1c5cec0 Remove http status code maps (#31350)
  87a676e Do not set vm.max_map_count when unnecessary (#31285)
  e5b7137 TEST: getCapturedRequestsAndClear should be atomic (#31312)
  0324103 Painless: Fix bug for static method calls on interfaces (#31348)
  d6d0727 QA: Fix resolution of default distribution (#31351)
  fcf1e41 Extract common http logic to server (#31311)
  6dd81ea Build: Fix the license in the pom zip and tar (#31336)
  8f886cd Treat ack timeout more like a publish timeout (#31303)
  9b29327 [ML] Add description to ML filters (#31330)
  f7a0caf SQL: Fix build on Java 10
  375d09c [TEST] Fix RemoteClusterClientTests#testEnsureWeReconnect
  4877cec More detailed tracing when writing metadata (#31319)
  bbfe1ec [Tests] Mutualize fixtures code in BaseHttpFixture (#31210)

Author: tlrx

build.gradle

@@ -53,9 +53,23 @@ subprojects {
   description = "Elasticsearch subproject ${project.path}"
 }
 
+apply plugin: 'nebula.info-scm'
+String licenseCommit
+if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
+  licenseCommit = scminfo.change ?: "master" // leniency for non git builds
+} else {
+  licenseCommit = "v${version}"
+}
+String elasticLicenseUrl = "https://raw.githubusercontent.com/elastic/elasticsearch/${licenseCommit}/licenses/ELASTIC-LICENSE.txt"
+
 subprojects {
+  // Default to the apache license
   project.ext.licenseName = 'The Apache Software License, Version 2.0'
   project.ext.licenseUrl = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+
+  // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
+  project.ext.elasticLicenseUrl = elasticLicenseUrl
+
   // we only use maven publish to add tasks for pom generation
   plugins.withType(MavenPublishPlugin).whenPluginAdded {
     publishing {

distribution/archives/build.gradle

@@ -228,6 +228,8 @@ subprojects {
   check.dependsOn checkNotice
 
   if (project.name == 'zip' || project.name == 'tar') {
+    project.ext.licenseName = 'Elastic License'
+    project.ext.licenseUrl = ext.elasticLicenseUrl
     task checkMlCppNotice {
       dependsOn buildDist, checkExtraction
       onlyIf toolExists

distribution/packages/src/deb/init.d/elasticsearch

@@ -122,7 +122,7 @@ case "$1" in
 		ulimit -l $MAX_LOCKED_MEMORY
 	fi
 
-	if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
+	if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count -a "$MAX_MAP_COUNT" -ge $(cat /proc/sys/vm/max_map_count) ]; then
 		sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
 	fi
 

distribution/packages/src/rpm/init.d/elasticsearch

@@ -90,7 +90,7 @@ start() {
     if [ -n "$MAX_LOCKED_MEMORY" ]; then
         ulimit -l $MAX_LOCKED_MEMORY
     fi
-    if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
+    if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count -a "$MAX_MAP_COUNT" -ge $(cat /proc/sys/vm/max_map_count) ]; then
         sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
     fi
 

docs/reference/setup/install/docker.asciidoc

@@ -279,6 +279,13 @@ docker build --tag=elasticsearch-custom .
 docker run -ti -v /usr/share/elasticsearch/data elasticsearch-custom
 --------------------------------------------
 
+Some plugins require additional security permissions. You have to explicitly accept
+them either by attaching a `tty` when you run the Docker image and accepting yes at 
+the prompts, or inspecting the security permissions separately and if you are 
+comfortable with them adding the `--batch` flag to the plugin install command.
+See {plugins}/_other_command_line_parameters.html[Plugin Management documentation]
+for more details.
+
 ===== D. Override the image's default https://docs.docker.com/engine/reference/run/#cmd-default-command-or-options[CMD]
 
 Options can be passed as command-line options to the {es} process by

modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java

@@ -376,7 +376,8 @@ private static MethodHandle lookupReferenceInternal(Definition definition, Looku
              ref.delegateClassName,
              ref.delegateInvokeType,
              ref.delegateMethodName,
-             ref.delegateMethodType
+             ref.delegateMethodType,
+             ref.isDelegateInterface ? 1 : 0
          );
          return callSite.dynamicInvoker().asType(MethodType.methodType(clazz.clazz, captures));
      }

modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.painless;
 
 import org.elasticsearch.painless.spi.Whitelist;
+import org.objectweb.asm.Opcodes;
 
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodHandles;
@@ -202,16 +203,28 @@ public MethodType getMethodType() {
 
         public void write(MethodWriter writer) {
             final org.objectweb.asm.Type type;
+            final Class<?> clazz;
             if (augmentation != null) {
                 assert java.lang.reflect.Modifier.isStatic(modifiers);
+                clazz = augmentation;
                 type = org.objectweb.asm.Type.getType(augmentation);
             } else {
+                clazz = owner.clazz;
                 type = owner.type;
             }
 
             if (java.lang.reflect.Modifier.isStatic(modifiers)) {
-                writer.invokeStatic(type, method);
-            } else if (java.lang.reflect.Modifier.isInterface(owner.clazz.getModifiers())) {
+                // invokeStatic assumes that the owner class is not an interface, so this is a
+                // special case for interfaces where the interface method boolean needs to be set to
+                // true to reference the appropriate class constant when calling a static interface
+                // method since java 8 did not check, but java 9 and 10 do
+                if (java.lang.reflect.Modifier.isInterface(clazz.getModifiers())) {
+                    writer.visitMethodInsn(Opcodes.INVOKESTATIC,
+                            type.getInternalName(), name, getMethodType().toMethodDescriptorString(), true);
+                } else {
+                    writer.invokeStatic(type, method);
+                }
+            } else if (java.lang.reflect.Modifier.isInterface(clazz.getModifiers())) {
                 writer.invokeInterface(type, method);
             } else {
                 writer.invokeVirtual(type, method);

modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java

@@ -66,6 +66,9 @@ public class FunctionRef {
     /** delegate method type method as type */
     public final Type delegateType;
 
+    /** whether a call is made on a delegate interface */
+    public final boolean isDelegateInterface;
+
     /**
      * Creates a new FunctionRef, which will resolve {@code type::call} from the whitelist.
      * @param definition the whitelist against which this script is being compiled
@@ -97,10 +100,13 @@ public FunctionRef(Class<?> expected, Method interfaceMethod, Method delegateMet
         // the Painless$Script class can be inferred if owner is null
         if (delegateMethod.owner == null) {
             delegateClassName = CLASS_NAME;
+            isDelegateInterface = false;
         } else if (delegateMethod.augmentation != null) {
             delegateClassName = delegateMethod.augmentation.getName();
+            isDelegateInterface = delegateMethod.augmentation.isInterface();
         } else {
             delegateClassName = delegateMethod.owner.clazz.getName();
+            isDelegateInterface = delegateMethod.owner.clazz.isInterface();
         }
 
         if ("<init>".equals(delegateMethod.name)) {
@@ -139,6 +145,7 @@ public FunctionRef(Class<?> expected,
         delegateInvokeType = H_INVOKESTATIC;
         this.delegateMethodName = delegateMethodName;
         this.delegateMethodType = delegateMethodType.dropParameterTypes(0, numCaptures);
+        isDelegateInterface = false;
 
         this.interfaceMethod = null;
         delegateMethod = null;

modules/lang-painless/src/main/java/org/elasticsearch/painless/LambdaBootstrap.java

@@ -188,6 +188,10 @@ private Capture(int count, Class<?> type) {
      * @param delegateMethodName The name of the method to be called in the Painless script class
      * @param delegateMethodType The type of method call in the Painless script class without
      *                           the captured types
+     * @param isDelegateInterface If the method to be called is owned by an interface where
+     *                            if the value is '1' if the delegate is an interface and '0'
+     *                            otherwise; note this is an int because the bootstrap method
+     *                            cannot convert constants to boolean
      * @return A {@link CallSite} linked to a factory method for creating a lambda class
      * that implements the expected functional interface
      * @throws LambdaConversionException Thrown when an illegal type conversion occurs at link time
@@ -200,7 +204,8 @@ public static CallSite lambdaBootstrap(
             String delegateClassName,
             int delegateInvokeType,
             String delegateMethodName,
-            MethodType delegateMethodType)
+            MethodType delegateMethodType,
+            int isDelegateInterface)
             throws LambdaConversionException {
         Loader loader = (Loader)lookup.lookupClass().getClassLoader();
         String lambdaClassName = Type.getInternalName(lookup.lookupClass()) + "$$Lambda" + loader.newLambdaIdentifier();
@@ -225,7 +230,7 @@ public static CallSite lambdaBootstrap(
 
         generateInterfaceMethod(cw, factoryMethodType, lambdaClassType, interfaceMethodName,
             interfaceMethodType, delegateClassType, delegateInvokeType,
-            delegateMethodName, delegateMethodType, captures);
+            delegateMethodName, delegateMethodType, isDelegateInterface == 1, captures);
 
         endLambdaClass(cw);
 
@@ -369,6 +374,7 @@ private static void generateInterfaceMethod(
             int delegateInvokeType,
             String delegateMethodName,
             MethodType delegateMethodType,
+            boolean isDelegateInterface,
             Capture[] captures)
             throws LambdaConversionException {
 
@@ -434,7 +440,7 @@ private static void generateInterfaceMethod(
         Handle delegateHandle =
             new Handle(delegateInvokeType, delegateClassType.getInternalName(),
                 delegateMethodName, delegateMethodType.toMethodDescriptorString(),
-                delegateInvokeType == H_INVOKEINTERFACE);
+                isDelegateInterface);
         iface.invokeDynamic(delegateMethodName, Type.getMethodType(interfaceMethodType
                 .toMethodDescriptorString()).getDescriptor(), DELEGATE_BOOTSTRAP_HANDLE,
                 delegateHandle);

modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java

@@ -141,8 +141,8 @@ public final class WriterConstants {
 
     /** invokedynamic bootstrap for lambda expression/method references */
     public static final MethodType LAMBDA_BOOTSTRAP_TYPE =
-            MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class,
-                                  MethodType.class, MethodType.class, String.class, int.class, String.class, MethodType.class);
+            MethodType.methodType(CallSite.class, MethodHandles.Lookup.class, String.class, MethodType.class,
+                    MethodType.class, String.class, int.class, String.class, MethodType.class, int.class);
     public static final Handle LAMBDA_BOOTSTRAP_HANDLE =
             new Handle(Opcodes.H_INVOKESTATIC, Type.getInternalName(LambdaBootstrap.class),
                 "lambdaBootstrap", LAMBDA_BOOTSTRAP_TYPE.toMethodDescriptorString(), false);

modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java

@@ -121,7 +121,8 @@ void write(MethodWriter writer, Globals globals) {
                 ref.delegateClassName,
                 ref.delegateInvokeType,
                 ref.delegateMethodName,
-                ref.delegateType
+                ref.delegateType,
+                ref.isDelegateInterface ? 1 : 0
             );
         }
     }

modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java

@@ -112,7 +112,8 @@ void write(MethodWriter writer, Globals globals) {
                 ref.delegateClassName,
                 ref.delegateInvokeType,
                 ref.delegateMethodName,
-                ref.delegateType
+                ref.delegateType,
+                ref.isDelegateInterface ? 1 : 0
             );
         } else {
             // TODO: don't do this: its just to cutover :)

modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java

@@ -222,7 +222,8 @@ void write(MethodWriter writer, Globals globals) {
                 ref.delegateClassName,
                 ref.delegateInvokeType,
                 ref.delegateMethodName,
-                ref.delegateType
+                ref.delegateType,
+                ref.isDelegateInterface ? 1 : 0
             );
         } else {
             // placeholder

modules/lang-painless/src/test/java/org/elasticsearch/painless/BasicExpressionTests.java

@@ -264,6 +264,11 @@ public void testNullSafeDeref() {
 //        assertEquals(null, exec("def a = ['thing': 'bar']; a.other?.cat?.dog = 'wombat'; return a.other?.cat?.dog"));
     }
 
+    // test to ensure static interface methods are called correctly
+    public void testStaticInterfaceMethod() {
+        assertEquals(4, exec("def values = [1, 4, 3, 2]; values.sort(Comparator.comparing(p -> p)); return values[3]"));
+    }
+
     private void assertMustBeNullable(String script) {
         Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> exec(script));
         assertEquals("Result of null safe operator must be nullable", e.getMessage());

modules/lang-painless/src/test/java/org/elasticsearch/painless/FunctionRefTests.java

@@ -184,6 +184,11 @@ public void testInterfaceDefaultMethodDef() {
                                  "def map = new HashMap(); f(map::getOrDefault)"));
     }
 
+    public void testInterfaceStaticMethod() {
+        assertEquals(-1, exec("Supplier get(Supplier supplier) { return supplier }" +
+                              "Supplier s = get(Comparator::naturalOrder); s.get().compare(1, 2)"));
+    }
+
     public void testMethodMissing() {
         Exception e = expectScriptThrows(IllegalArgumentException.class, () -> {
             exec("List l = [2, 1]; l.sort(Integer::bogus); return l.get(0);");

modules/repository-url/build.gradle

@@ -23,12 +23,6 @@ esplugin {
     classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
 }
 
-forbiddenApisTest {
-    // we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
-    bundledSignatures -= 'jdk-non-portable'
-    bundledSignatures += 'jdk-internal'
-}
-
 // This directory is shared between two URL repositories and one FS repository in YAML integration tests
 File repositoryDir = new File(project.buildDir, "shared-repository")