Merge remote-tracking branch 'elastic/master' into settings-version

* elastic/master:
  Mute PartitionedRoutingIT#testShrinking on Windows
  Mute testToQuery test
  [TEST] Make sure there are shards started so that `ESIntegTestCase#assertSameDocIdsOnShards()` does not fail with shard not found.
  Change shard changes api's threadpool from get to search (elastic#34421)
  Update TESTING.asciidoc title (elastic#34401)
  Tests: Fix DateFormatter equals tests with locale (elastic#34435)
  Docs: Remove unnecessary qualifier from wildcard import note (elastic#34419)
  CCR/TEST: AwaitsFix testFailOverOnFollower
  [Painless] Add a Map for java names to classes for use in the custom classloader (elastic#34424)
  TEST: Fix indentation in FullClusterRestartIT (elastic#34420)
  [WIP] Ingest Attachement: Upgrade tika to v1.19.1 (elastic#33896)
  NETWORKING: Upgrade Netty to 4.1.30 (elastic#34417)
  Allow an AuthenticationResult to return metadata (elastic#34382)
  [ML] Add an ingest pipeline definition to structure finder (elastic#34350)
  Handle pre-6.x time fields (elastic#34373)
  ListenableFuture should preserve ThreadContext (elastic#34394)

Author: jasontedor

CONTRIBUTING.md

@@ -161,7 +161,7 @@ Please follow these formatting guidelines:
 * Line width is 140 characters
 * The rest is left to Java coding standards
 * Disable “auto-format on save” to prevent unnecessary format changes. This makes reviews much harder as it generates unnecessary formatting changes. If your IDE supports formatting only modified chunks that is fine to do.
-* Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. Please attempt to tame your IDE so it doesn't make them and please send a PR against this document with instructions for your IDE if it doesn't contain them.
+* Wildcard imports (`import foo.bar.baz.*`) are forbidden and will cause the build to fail. This can be done automatically by your IDE:
  * Eclipse: `Preferences->Java->Code Style->Organize Imports`. There are two boxes labeled "`Number of (static )? imports needed for .*`". Set their values to 99999 or some other absurdly high value.
  * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value.
 * Don't worry too much about import order. Try not to change it but don't worry about fighting your IDE to stop it from doing so.

TESTING.asciidoc

@@ -1,4 +1,4 @@
-[[Testing Framework Cheatsheet]]
+[[TestingFrameworkCheatsheet]]
 = Testing
 
 [partintro]

buildSrc/version.properties

@@ -15,6 +15,8 @@ slf4j             = 1.6.2
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 4.5.1
 
+netty             = 4.1.30.Final
+
 # test dependencies
 randomizedrunner  = 2.7.0
 junit             = 4.12

docs/reference/ml/apis/find-file-structure.asciidoc

@@ -613,6 +613,20 @@ If the request does not encounter errors, you receive the following result:
       "type" : "double"
     }
   },
+  "ingest_pipeline" : {
+    "description" : "Ingest pipeline created by file structure finder",
+    "processors" : [
+      {
+        "date" : {
+          "field" : "tpep_pickup_datetime",
+          "timezone" : "{{ beat.timezone }}",
+          "formats" : [
+            "YYYY-MM-dd HH:mm:ss"
+          ]
+        }
+      }
+    ]
+  },
   "field_stats" : {
     "DOLocationID" : {
       "count" : 19998,
@@ -1366,6 +1380,33 @@ this:
       "type" : "text"
     }
   },
+  "ingest_pipeline" : {
+    "description" : "Ingest pipeline created by file structure finder",
+    "processors" : [
+      {
+        "grok" : {
+          "field" : "message",
+          "patterns" : [
+            "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel}.*"
+          ]
+        }
+      },
+      {
+        "date" : {
+          "field" : "timestamp",
+          "timezone" : "{{ beat.timezone }}",
+          "formats" : [
+            "ISO8601"
+          ]
+        }
+      },
+      {
+        "remove" : {
+          "field" : "timestamp"
+        }
+      }
+    ]
+  },
   "field_stats" : {
     "loglevel" : {
       "count" : 53,
@@ -1499,6 +1540,33 @@ this:
       "type" : "keyword"
     }
   },
+  "ingest_pipeline" : {
+    "description" : "Ingest pipeline created by file structure finder",
+    "processors" : [
+      {
+        "grok" : {
+          "field" : "message",
+          "patterns" : [
+            "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{LOGLEVEL:loglevel} *\\]\\[%{JAVACLASS:class} *\\] \\[%{HOSTNAME:node}\\] %{JAVALOGMESSAGE:message}"
+          ]
+        }
+      },
+      {
+        "date" : {
+          "field" : "timestamp",
+          "timezone" : "{{ beat.timezone }}",
+          "formats" : [
+            "ISO8601"
+          ]
+        }
+      },
+      {
+        "remove" : {
+          "field" : "timestamp"
+        }
+      }
+    ]
+  },
   "field_stats" : { <2>
     "class" : {
       "count" : 53,

modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java

@@ -96,7 +96,7 @@ public Class<?> findClass(String name) throws ClassNotFoundException {
             if (found != null) {
                 return found;
             }
-            found = painlessLookup.canonicalTypeNameToType(name.replace('$', '.'));
+            found = painlessLookup.javaClassNameToClass(name);
 
             return found != null ? found : super.findClass(name);
         }

modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java

@@ -34,29 +34,39 @@
 
 public final class PainlessLookup {
 
+    private final Map<String, Class<?>> javaClassNamesToClasses;
     private final Map<String, Class<?>> canonicalClassNamesToClasses;
     private final Map<Class<?>, PainlessClass> classesToPainlessClasses;
 
     private final Map<String, PainlessMethod> painlessMethodKeysToImportedPainlessMethods;
     private final Map<String, PainlessClassBinding> painlessMethodKeysToPainlessClassBindings;
 
-    PainlessLookup(Map<String, Class<?>> canonicalClassNamesToClasses, Map<Class<?>, PainlessClass> classesToPainlessClasses,
+    PainlessLookup(
+            Map<String, Class<?>> javaClassNamesToClasses,
+            Map<String, Class<?>> canonicalClassNamesToClasses,
+            Map<Class<?>, PainlessClass> classesToPainlessClasses,
             Map<String, PainlessMethod> painlessMethodKeysToImportedPainlessMethods,
             Map<String, PainlessClassBinding> painlessMethodKeysToPainlessClassBindings) {
 
+        Objects.requireNonNull(javaClassNamesToClasses);
         Objects.requireNonNull(canonicalClassNamesToClasses);
         Objects.requireNonNull(classesToPainlessClasses);
 
         Objects.requireNonNull(painlessMethodKeysToImportedPainlessMethods);
         Objects.requireNonNull(painlessMethodKeysToPainlessClassBindings);
 
+        this.javaClassNamesToClasses = javaClassNamesToClasses;
         this.canonicalClassNamesToClasses = Collections.unmodifiableMap(canonicalClassNamesToClasses);
         this.classesToPainlessClasses = Collections.unmodifiableMap(classesToPainlessClasses);
 
         this.painlessMethodKeysToImportedPainlessMethods = Collections.unmodifiableMap(painlessMethodKeysToImportedPainlessMethods);
         this.painlessMethodKeysToPainlessClassBindings = Collections.unmodifiableMap(painlessMethodKeysToPainlessClassBindings);
     }
 
+    public Class<?> javaClassNameToClass(String javaClassName) {
+        return javaClassNamesToClasses.get(javaClassName);
+    }
+
     public boolean isValidCanonicalClassName(String canonicalClassName) {
         Objects.requireNonNull(canonicalClassName);
 

modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookupBuilder.java

@@ -120,13 +120,23 @@ public static PainlessLookup buildFromWhitelists(List<Whitelist> whitelists) {
         return painlessLookupBuilder.build();
     }
 
+    // javaClassNamesToClasses is all the classes that need to be available to the custom classloader
+    // including classes used as part of imported methods and class bindings but not necessarily whitelisted
+    // individually. The values of javaClassNamesToClasses are a superset of the values of
+    // canonicalClassNamesToClasses.
+    private final Map<String, Class<?>> javaClassNamesToClasses;
+    // canonicalClassNamesToClasses is all the whitelisted classes available in a Painless script including
+    // classes with imported canonical names but does not include classes from imported methods or class
+    // bindings unless also whitelisted separately. The values of canonicalClassNamesToClasses are a subset
+    // of the values of javaClassNamesToClasses.
     private final Map<String, Class<?>> canonicalClassNamesToClasses;
     private final Map<Class<?>, PainlessClassBuilder> classesToPainlessClassBuilders;
 
     private final Map<String, PainlessMethod> painlessMethodKeysToImportedPainlessMethods;
     private final Map<String, PainlessClassBinding> painlessMethodKeysToPainlessClassBindings;
 
     public PainlessLookupBuilder() {
+        javaClassNamesToClasses = new HashMap<>();
         canonicalClassNamesToClasses = new HashMap<>();
         classesToPainlessClassBuilders = new HashMap<>();
 
@@ -189,7 +199,16 @@ public void addPainlessClass(Class<?> clazz, boolean importClassName) {
             throw new IllegalArgumentException("invalid class name [" + canonicalClassName + "]");
         }
 
-        Class<?> existingClass = canonicalClassNamesToClasses.get(canonicalClassName);
+        Class<?> existingClass = javaClassNamesToClasses.get(clazz.getName());
+
+        if (existingClass == null) {
+            javaClassNamesToClasses.put(clazz.getName(), clazz);
+        } else if (existingClass != clazz) {
+            throw new IllegalArgumentException("class [" + canonicalClassName + "] " +
+                    "cannot represent multiple java classes with the same name from different class loaders");
+        }
+
+        existingClass = canonicalClassNamesToClasses.get(canonicalClassName);
 
         if (existingClass != null && existingClass != clazz) {
             throw new IllegalArgumentException("class [" + canonicalClassName + "] " +
@@ -685,6 +704,14 @@ public void addImportedPainlessMethod(Class<?> targetClass, String methodName, C
         }
 
         String targetCanonicalClassName = typeToCanonicalTypeName(targetClass);
+        Class<?> existingTargetClass = javaClassNamesToClasses.get(targetClass.getName());
+
+        if (existingTargetClass == null) {
+            javaClassNamesToClasses.put(targetClass.getName(), targetClass);
+        } else if (existingTargetClass != targetClass) {
+            throw new IllegalArgumentException("class [" + targetCanonicalClassName + "] " +
+                    "cannot represent multiple java classes with the same name from different class loaders");
+        }
 
         if (METHOD_NAME_PATTERN.matcher(methodName).matches() == false) {
             throw new IllegalArgumentException(
@@ -818,6 +845,14 @@ public void addPainlessClassBinding(Class<?> targetClass, String methodName, Cla
         }
 
         String targetCanonicalClassName = typeToCanonicalTypeName(targetClass);
+        Class<?> existingTargetClass = javaClassNamesToClasses.get(targetClass.getName());
+
+        if (existingTargetClass == null) {
+            javaClassNamesToClasses.put(targetClass.getName(), targetClass);
+        } else if (existingTargetClass != targetClass) {
+            throw new IllegalArgumentException("class [" + targetCanonicalClassName + "] " +
+                    "cannot represent multiple java classes with the same name from different class loaders");
+        }
 
         Constructor<?>[] javaConstructors = targetClass.getConstructors();
         Constructor<?> javaConstructor = null;
@@ -952,7 +987,23 @@ public PainlessLookup build() {
             classesToPainlessClasses.put(painlessClassBuilderEntry.getKey(), painlessClassBuilderEntry.getValue().build());
         }
 
-        return new PainlessLookup(canonicalClassNamesToClasses, classesToPainlessClasses,
+        if (javaClassNamesToClasses.values().containsAll(canonicalClassNamesToClasses.values()) == false) {
+            throw new IllegalArgumentException("the values of java class names to classes " +
+                    "must be a superset of the values of canonical class names to classes");
+        }
+
+        if (javaClassNamesToClasses.values().containsAll(classesToPainlessClasses.keySet()) == false) {
+            throw new IllegalArgumentException("the values of java class names to classes " +
+                    "must be a superset of the keys of classes to painless classes");
+        }
+
+        if (canonicalClassNamesToClasses.values().containsAll(classesToPainlessClasses.keySet()) == false ||
+                classesToPainlessClasses.keySet().containsAll(canonicalClassNamesToClasses.values()) == false) {
+            throw new IllegalArgumentException("the values of canonical class names to classes "  +
+                    "must have the same classes as the keys of classes to painless classes");
+        }
+
+        return new PainlessLookup(javaClassNamesToClasses, canonicalClassNamesToClasses, classesToPainlessClasses,
                 painlessMethodKeysToImportedPainlessMethods, painlessMethodKeysToPainlessClassBindings);
     }
 

modules/transport-netty4/build.gradle

@@ -34,13 +34,13 @@ compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-tr
 
 dependencies {
     // network stack
-    compile "io.netty:netty-buffer:4.1.29.Final"
-    compile "io.netty:netty-codec:4.1.29.Final"
-    compile "io.netty:netty-codec-http:4.1.29.Final"
-    compile "io.netty:netty-common:4.1.29.Final"
-    compile "io.netty:netty-handler:4.1.29.Final"
-    compile "io.netty:netty-resolver:4.1.29.Final"
-    compile "io.netty:netty-transport:4.1.29.Final"
+    compile "io.netty:netty-buffer:${versions.netty}"
+    compile "io.netty:netty-codec:${versions.netty}"
+    compile "io.netty:netty-codec-http:${versions.netty}"
+    compile "io.netty:netty-common:${versions.netty}"
+    compile "io.netty:netty-handler:${versions.netty}"
+    compile "io.netty:netty-resolver:${versions.netty}"
+    compile "io.netty:netty-transport:${versions.netty}"
 }
 
 dependencyLicenses {
@@ -111,6 +111,7 @@ thirdPartyAudit.excludes = [
         // from io.netty.util.internal.logging.InternalLoggerFactory (netty) - it's optional
         'org.slf4j.Logger',
         'org.slf4j.LoggerFactory',
+        'org.slf4j.spi.LocationAwareLogger',
 
         'com.google.protobuf.ExtensionRegistryLite',
         'com.google.protobuf.MessageLiteOrBuilder',
@@ -145,6 +146,7 @@ thirdPartyAudit.excludes = [
         'io.netty.util.internal.PlatformDependent0$1',
         'io.netty.util.internal.PlatformDependent0$2',
         'io.netty.util.internal.PlatformDependent0$3',
+        'io.netty.util.internal.PlatformDependent0$5',
         'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueConsumerNodeRef',
         'io.netty.util.internal.shaded.org.jctools.queues.BaseLinkedQueueProducerNodeRef',
         'io.netty.util.internal.shaded.org.jctools.queues.BaseMpscLinkedArrayQueueColdProducerFields',

modules/transport-netty4/licenses/netty-buffer-4.1.29.Final.jar.sha1

@@ -0,0 +1 @@
+597adb653306470fb3ec1af3c0f3f30a37b1310a

modules/transport-netty4/licenses/netty-buffer-4.1.30.Final.jar.sha1

@@ -0,0 +1 @@
+515c8f609aaca28a94f984d89a9667dd3359c1b1

modules/transport-netty4/licenses/netty-codec-4.1.29.Final.jar.sha1

@@ -0,0 +1 @@
+1384c630e8a0eeef33ad12a28791dce6e1d8767c

modules/transport-netty4/licenses/netty-codec-4.1.30.Final.jar.sha1

@@ -0,0 +1 @@
+5dca0c34d8f38af51a2398614e81888f51cf811a

modules/transport-netty4/licenses/netty-codec-http-4.1.29.Final.jar.sha1

@@ -0,0 +1 @@
+ecc076332ed103411347f4806a44ee32d9d9cb5f

modules/transport-netty4/licenses/netty-codec-http-4.1.30.Final.jar.sha1

@@ -0,0 +1 @@
+5106fd687066ffd712e5295d32af4e2ac6482613

modules/transport-netty4/licenses/netty-common-4.1.29.Final.jar.sha1

@@ -0,0 +1 @@
+3d27bb432a3b125167ac161b26415ad29ec17f02

modules/transport-netty4/licenses/netty-common-4.1.30.Final.jar.sha1

@@ -0,0 +1,26 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.example.painlesswhitelist;
+
+public class ExampleStaticMethodClass {
+    public static int exampleAddInts(int x, int y) {
+        return x + y;
+    }
+}

modules/transport-netty4/licenses/netty-handler-4.1.29.Final.jar.sha1

@@ -39,4 +39,8 @@ class java.lang.String {
   # existing classes can be "augmented" to have additional methods, which take the object
   # to operate on as the first argument to a static method
   int org.elasticsearch.example.painlesswhitelist.ExampleWhitelistedClass toInt()
+}
+
+static_import {
+  int exampleAddInts(int, int) from_class org.elasticsearch.example.painlesswhitelist.ExampleStaticMethodClass
 }