Merge branch 'master' into xpack-license

* master:
  DOCS Audit event attributes in new format (elastic#35510)
  Scripting: Actually add joda time back to whitelist (elastic#35965)
  [DOCS] fix HLRC ILM doc misreferenced tag
  Add realm information for Authenticate API (elastic#35648)
  [ILM] add HLRC docs to remove-policy-from-index (elastic#35759)
  [Rollup] Update serialization version after backport
  [Rollup] Add more diagnostic stats to job (elastic#35471)
  Build: Fix gradle build for Mac OS (elastic#35968)
  Adds deprecation logging to ScriptDocValues#getValues. (elastic#34279)
  [Monitoring] Make Exporters Async (elastic#35765)
  [ILM] reduce time restriction on IndexLifecycleExplainResponse (elastic#35954)
  Remove use of AbstractComponent in xpack (elastic#35394)
  Deprecate types in search and multi search templates. (elastic#35669)
  Remove fromXContent from IndexUpgradeInfoResponse (elastic#35934)

Author: jasontedor

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -910,7 +910,7 @@ class BuildPlugin implements Plugin<Project> {
                 args '-n', 'hw.physicalcpu'
                 standardOutput = stdout
             }
-            return stdout.toString('UTF-8')
+            return stdout.toString('UTF-8').trim();
         }
         return 'auto';
     }

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/IndexLifecycleExplainResponse.java

@@ -124,15 +124,12 @@ private IndexLifecycleExplainResponse(String index, boolean managedByILM, String
                 throw new IllegalArgumentException("[" + POLICY_NAME_FIELD.getPreferredName() + "] cannot be null for managed index");
             }
             // check to make sure that step details are either all null or all set.
-            long numNull = Stream.of(phase, action, step, phaseTime, actionTime, stepTime).filter(Objects::isNull).count();
-            if (numNull > 0 && numNull < 6) {
+            long numNull = Stream.of(phase, action, step).filter(Objects::isNull).count();
+            if (numNull > 0 && numNull < 3) {
                 throw new IllegalArgumentException("managed index response must have complete step details [" +
                     PHASE_FIELD.getPreferredName() + "=" + phase + ", " +
-                    PHASE_TIME_FIELD.getPreferredName() + "=" + phaseTime + ", " +
                     ACTION_FIELD.getPreferredName() + "=" + action + ", " +
-                    ACTION_TIME_FIELD.getPreferredName() + "=" + actionTime + ", " +
-                    STEP_FIELD.getPreferredName() + "=" + step + ", " +
-                    STEP_TIME_FIELD.getPreferredName() + "=" + stepTime + "]");
+                    STEP_FIELD.getPreferredName() + "=" + step + "]");
             }
         } else {
             if (policyName != null || lifecycleDate != null || phase != null || action != null || step != null || failedStep != null

client/rest-high-level/src/main/java/org/elasticsearch/client/rollup/GetRollupJobResponse.java

@@ -26,15 +26,15 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
-import java.util.Objects;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static java.util.Collections.unmodifiableList;
 import static java.util.stream.Collectors.joining;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
+import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
  * Response from rollup's get jobs api.
@@ -51,6 +51,12 @@ public class GetRollupJobResponse {
     static final ParseField STATE = new ParseField("job_state");
     static final ParseField CURRENT_POSITION = new ParseField("current_position");
     static final ParseField UPGRADED_DOC_ID = new ParseField("upgraded_doc_id");
+    static final ParseField INDEX_TIME_IN_MS = new ParseField("index_time_in_ms");
+    static final ParseField SEARCH_TIME_IN_MS = new ParseField("search_time_in_ms");
+    static final ParseField INDEX_TOTAL = new ParseField("index_total");
+    static final ParseField SEARCH_TOTAL = new ParseField("search_total");
+    static final ParseField SEARCH_FAILURES = new ParseField("search_failures");
+    static final ParseField INDEX_FAILURES = new ParseField("index_failures");
 
     private List<JobWrapper> jobs;
 
@@ -181,12 +187,25 @@ public static class RollupIndexerJobStats {
         private final long numInputDocuments;
         private final long numOuputDocuments;
         private final long numInvocations;
-
-        RollupIndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations) {
+        private long indexTime;
+        private long indexTotal;
+        private long searchTime;
+        private long searchTotal;
+        private long indexFailures;
+        private long searchFailures;
+
+        RollupIndexerJobStats(long numPages, long numInputDocuments, long numOuputDocuments, long numInvocations,
+                              long indexTime, long indexTotal, long searchTime, long searchTotal, long indexFailures, long searchFailures) {
             this.numPages = numPages;
             this.numInputDocuments = numInputDocuments;
             this.numOuputDocuments = numOuputDocuments;
             this.numInvocations = numInvocations;
+            this.indexTime = indexTime;
+            this.indexTotal = indexTotal;
+            this.searchTime = searchTime;
+            this.searchTotal = searchTotal;
+            this.indexFailures = indexFailures;
+            this.searchFailures = searchFailures;
         }
 
         /**
@@ -217,15 +236,65 @@ public long getOutputDocuments() {
             return numOuputDocuments;
         }
 
+        /**
+         * Number of failures that have occurred during the bulk indexing phase of Rollup
+         */
+        public long getIndexFailures() {
+            return indexFailures;
+        }
+
+        /**
+         * Number of failures that have occurred during the search phase of Rollup
+         */
+        public long getSearchFailures() {
+            return searchFailures;
+        }
+
+        /**
+         * Returns the time spent indexing (cumulative) in milliseconds
+         */
+        public long getIndexTime() {
+            return indexTime;
+        }
+
+        /**
+         * Returns the time spent searching (cumulative) in milliseconds
+         */
+        public long getSearchTime() {
+            return searchTime;
+        }
+
+        /**
+         * Returns the total number of indexing requests that have been sent by the rollup job
+         * (Note: this is not the number of _documents_ that have been indexed)
+         */
+        public long getIndexTotal() {
+            return indexTotal;
+        }
+
+        /**
+         * Returns the total number of search requests that have been sent by the rollup job
+         */
+        public long getSearchTotal() {
+            return searchTotal;
+        }
+
         private static final ConstructingObjectParser<RollupIndexerJobStats, Void> PARSER = new ConstructingObjectParser<>(
                 STATS.getPreferredName(),
                 true,
-                args -> new RollupIndexerJobStats((long) args[0], (long) args[1], (long) args[2], (long) args[3]));
+                args -> new RollupIndexerJobStats((long) args[0], (long) args[1], (long) args[2], (long) args[3],
+                    (long) args[4], (long) args[5], (long) args[6], (long) args[7], (long) args[8], (long) args[9]));
         static {
             PARSER.declareLong(constructorArg(), NUM_PAGES);
             PARSER.declareLong(constructorArg(), NUM_INPUT_DOCUMENTS);
             PARSER.declareLong(constructorArg(), NUM_OUTPUT_DOCUMENTS);
             PARSER.declareLong(constructorArg(), NUM_INVOCATIONS);
+            PARSER.declareLong(constructorArg(), INDEX_TIME_IN_MS);
+            PARSER.declareLong(constructorArg(), INDEX_TOTAL);
+            PARSER.declareLong(constructorArg(), SEARCH_TIME_IN_MS);
+            PARSER.declareLong(constructorArg(), SEARCH_TOTAL);
+            PARSER.declareLong(constructorArg(), INDEX_FAILURES);
+            PARSER.declareLong(constructorArg(), SEARCH_FAILURES);
         }
 
         @Override
@@ -234,22 +303,35 @@ public boolean equals(Object other) {
             if (other == null || getClass() != other.getClass()) return false;
             RollupIndexerJobStats that = (RollupIndexerJobStats) other;
             return Objects.equals(this.numPages, that.numPages)
-                    && Objects.equals(this.numInputDocuments, that.numInputDocuments)
-                    && Objects.equals(this.numOuputDocuments, that.numOuputDocuments)
-                    && Objects.equals(this.numInvocations, that.numInvocations);
+                && Objects.equals(this.numInputDocuments, that.numInputDocuments)
+                && Objects.equals(this.numOuputDocuments, that.numOuputDocuments)
+                && Objects.equals(this.numInvocations, that.numInvocations)
+                && Objects.equals(this.indexTime, that.indexTime)
+                && Objects.equals(this.searchTime, that.searchTime)
+                && Objects.equals(this.indexFailures, that.indexFailures)
+                && Objects.equals(this.searchFailures, that.searchFailures)
+                && Objects.equals(this.searchTotal, that.searchTotal)
+                && Objects.equals(this.indexTotal, that.indexTotal);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations);
+            return Objects.hash(numPages, numInputDocuments, numOuputDocuments, numInvocations,
+                indexTime, searchTime, indexFailures, searchFailures, searchTotal, indexTotal);
         }
 
         @Override
         public final String toString() {
             return "{pages=" + numPages
                     + ", input_docs=" + numInputDocuments
                     + ", output_docs=" + numOuputDocuments
-                    + ", invocations=" + numInvocations + "}";
+                    + ", invocations=" + numInvocations
+                    + ", index_failures=" + indexFailures
+                    + ", search_failures=" + searchFailures
+                    + ", index_time_in_ms=" + indexTime
+                    + ", index_total=" + indexTotal
+                    + ", search_time_in_ms=" + searchTime
+                    + ", search_total=" + searchTotal+ "}";
         }
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/security/AuthenticateResponse.java

@@ -46,27 +46,43 @@ public final class AuthenticateResponse {
     static final ParseField FULL_NAME = new ParseField("full_name");
     static final ParseField EMAIL = new ParseField("email");
     static final ParseField ENABLED = new ParseField("enabled");
+    static final ParseField AUTHENTICATION_REALM = new ParseField("authentication_realm");
+    static final ParseField LOOKUP_REALM = new ParseField("lookup_realm");
+    static final ParseField REALM_NAME = new ParseField("name");
+    static final ParseField REALM_TYPE = new ParseField("type");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<AuthenticateResponse, Void> PARSER = new ConstructingObjectParser<>(
             "client_security_authenticate_response",
             a -> new AuthenticateResponse(new User((String) a[0], ((List<String>) a[1]), (Map<String, Object>) a[2],
-                    (String) a[3], (String) a[4]), (Boolean) a[5]));
+                (String) a[3], (String) a[4]), (Boolean) a[5], (RealmInfo) a[6], (RealmInfo) a[7]));
     static {
+        final ConstructingObjectParser<RealmInfo, Void> realmInfoParser = new ConstructingObjectParser<>("realm_info",
+            a -> new RealmInfo((String) a[0], (String) a[1]));
+        realmInfoParser.declareString(constructorArg(), REALM_NAME);
+        realmInfoParser.declareString(constructorArg(), REALM_TYPE);
         PARSER.declareString(constructorArg(), USERNAME);
         PARSER.declareStringArray(constructorArg(), ROLES);
         PARSER.<Map<String, Object>>declareObject(constructorArg(), (parser, c) -> parser.map(), METADATA);
         PARSER.declareStringOrNull(optionalConstructorArg(), FULL_NAME);
         PARSER.declareStringOrNull(optionalConstructorArg(), EMAIL);
         PARSER.declareBoolean(constructorArg(), ENABLED);
+        PARSER.declareObject(constructorArg(), realmInfoParser, AUTHENTICATION_REALM);
+        PARSER.declareObject(constructorArg(), realmInfoParser, LOOKUP_REALM);
     }
 
     private final User user;
     private final boolean enabled;
+    private final RealmInfo authenticationRealm;
+    private final RealmInfo lookupRealm;
 
-    public AuthenticateResponse(User user, boolean enabled) {
+
+    public AuthenticateResponse(User user, boolean enabled, RealmInfo authenticationRealm,
+                                RealmInfo lookupRealm) {
         this.user = user;
         this.enabled = enabled;
+        this.authenticationRealm = authenticationRealm;
+        this.lookupRealm = lookupRealm;
     }
 
     /**
@@ -85,25 +101,69 @@ public boolean enabled() {
         return enabled;
     }
 
+    /**
+     * @return the realm that authenticated the user
+     */
+    public RealmInfo getAuthenticationRealm() {
+        return authenticationRealm;
+    }
+
+    /**
+     * @return the realm where the user information was looked up
+     */
+    public RealmInfo getLookupRealm() {
+        return lookupRealm;
+    }
+
     @Override
     public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-        final AuthenticateResponse that = (AuthenticateResponse) o;
-        return user.equals(that.user) && enabled == that.enabled;
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        AuthenticateResponse that = (AuthenticateResponse) o;
+        return enabled == that.enabled &&
+            Objects.equals(user, that.user) &&
+            Objects.equals(authenticationRealm, that.authenticationRealm) &&
+            Objects.equals(lookupRealm, that.lookupRealm);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(user, enabled);
+        return Objects.hash(user, enabled, authenticationRealm, lookupRealm);
     }
 
     public static AuthenticateResponse fromXContent(XContentParser parser) throws IOException {
         return PARSER.parse(parser, null);
     }
 
+    public static class RealmInfo {
+        private String name;
+        private String type;
+
+        RealmInfo(String name, String type) {
+            this.name = name;
+            this.type = type;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String getType() {
+            return type;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            RealmInfo realmInfo = (RealmInfo) o;
+            return Objects.equals(name, realmInfo.name) &&
+                Objects.equals(type, realmInfo.type);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(name, type);
+        }
+    }
 }