Merge remote-tracking branch 'origin/master' into reindex_v2

Author: henningandersen

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -78,6 +78,7 @@ import org.gradle.authentication.http.HttpHeaderAuthentication
 import org.gradle.external.javadoc.CoreJavadocOptions
 import org.gradle.internal.jvm.Jvm
 import org.gradle.language.base.plugins.LifecycleBasePlugin
+import org.gradle.process.CommandLineArgumentProvider
 import org.gradle.process.ExecResult
 import org.gradle.process.ExecSpec
 import org.gradle.util.GradleVersion
@@ -834,10 +835,9 @@ class BuildPlugin implements Plugin<Project> {
 
                 test.jvmArgs "-Xmx${System.getProperty('tests.heap.size', '512m')}",
                         "-Xms${System.getProperty('tests.heap.size', '512m')}",
-                        '-XX:+HeapDumpOnOutOfMemoryError',
-                        "-XX:HeapDumpPath=$heapdumpDir",
                         '--illegal-access=warn'
 
+                test.jvmArgumentProviders.add({ ['-XX:+HeapDumpOnOutOfMemoryError', "-XX:HeapDumpPath=$heapdumpDir"] } as CommandLineArgumentProvider)
 
                 if (System.getProperty('tests.jvm.argline')) {
                     test.jvmArgs System.getProperty('tests.jvm.argline').split(" ")
@@ -848,8 +848,7 @@ class BuildPlugin implements Plugin<Project> {
                 }
 
                 // we use './temp' since this is per JVM and tests are forbidden from writing to CWD
-                test.systemProperties 'gradle.dist.lib': new File(project.class.location.toURI()).parent,
-                        'java.io.tmpdir': './temp',
+                test.systemProperties 'java.io.tmpdir': './temp',
                         'java.awt.headless': 'true',
                         'tests.gradle': 'true',
                         'tests.artifact': project.name,
@@ -865,6 +864,7 @@ class BuildPlugin implements Plugin<Project> {
                 }
 
                 // don't track these as inputs since they contain absolute paths and break cache relocatability
+                nonInputProperties.systemProperty('gradle.dist.lib', new File(project.class.location.toURI()).parent)
                 nonInputProperties.systemProperty('gradle.worker.jar', "${project.gradle.getGradleUserHomeDir()}/caches/${project.gradle.gradleVersion}/workerMain/gradle-worker.jar")
                 nonInputProperties.systemProperty('gradle.user.home', project.gradle.getGradleUserHomeDir())
 

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -283,7 +283,7 @@ private static TaskProvider<?> configureDistroTest(Project project, Provider<Dir
     }
 
     private static TaskProvider<BatsTestTask> configureBatsTest(Project project, String type, Provider<Directory> distributionsDir,
-                                                                TaskProvider<Copy> copyPackagingArchives, Object... deps) {
+                                                                Object... deps) {
         return project.getTasks().register("destructiveBatsTest." + type, BatsTestTask.class,
             t -> {
                 Directory batsDir = project.getLayout().getProjectDirectory().dir("bats");

buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java

@@ -7,7 +7,6 @@
 import org.gradle.api.Project;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
 import org.gradle.internal.jvm.Jvm;
-import org.gradle.process.ExecResult;
 
 import java.io.BufferedReader;
 import java.io.ByteArrayOutputStream;
@@ -16,18 +15,19 @@
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-
 public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private static final String GLOBAL_INFO_EXTENSION_NAME = "globalInfo";
     private static Integer _defaultParallel = null;
@@ -46,8 +46,6 @@ public void apply(Project project) {
         File compilerJavaHome = findCompilerJavaHome();
         File runtimeJavaHome = findRuntimeJavaHome(compilerJavaHome);
 
-        Object gitRevisionResolver = createGitRevisionResolver(project);
-
         final List<JavaHome> javaVersions = new ArrayList<>();
         for (int version = 8; version <= Integer.parseInt(minimumCompilerVersion.getMajorVersion()); version++) {
             if (System.getenv(getJavaHomeEnvVarName(Integer.toString(version))) != null) {
@@ -95,7 +93,7 @@ public void apply(Project project) {
             ext.set("minimumCompilerVersion", minimumCompilerVersion);
             ext.set("minimumRuntimeVersion", minimumRuntimeVersion);
             ext.set("gradleJavaVersion", Jvm.current().getJavaVersion());
-            ext.set("gitRevision", gitRevisionResolver);
+            ext.set("gitRevision", gitRevision(project));
             ext.set("buildDate", ZonedDateTime.now(ZoneOffset.UTC));
         });
     }
@@ -206,35 +204,65 @@ private static int findDefaultParallel(Project project) {
         return _defaultParallel;
     }
 
-    private Object createGitRevisionResolver(final Project project) {
-        return new Object() {
-            private final AtomicReference<String> gitRevision = new AtomicReference<>();
-
-            @Override
-            public String toString() {
-                if (gitRevision.get() == null) {
-                    final ByteArrayOutputStream stdout = new ByteArrayOutputStream();
-                    final ByteArrayOutputStream stderr = new ByteArrayOutputStream();
-                    final ExecResult result = project.exec(spec -> {
-                        spec.setExecutable("git");
-                        spec.setArgs(Arrays.asList("rev-parse", "HEAD"));
-                        spec.setStandardOutput(stdout);
-                        spec.setErrorOutput(stderr);
-                        spec.setIgnoreExitValue(true);
-                    });
-
-                    final String revision;
-                    if (result.getExitValue() != 0) {
-                        revision = "unknown";
-                    } else {
-                        revision = stdout.toString(UTF_8).trim();
-                    }
-                    this.gitRevision.compareAndSet(null, revision);
+    private String gitRevision(final Project project) {
+        try {
+            /*
+             * We want to avoid forking another process to run git rev-parse HEAD. Instead, we will read the refs manually. The
+             * documentation for this follows from https://git-scm.com/docs/gitrepository-layout and https://git-scm.com/docs/git-worktree.
+             *
+             * There are two cases to consider:
+             *  - a plain repository with .git directory at the root of the working tree
+             *  - a worktree with a plain text .git file at the root of the working tree
+             *
+             * In each case, our goal is to parse the HEAD file to get either a ref or a bare revision (in the case of being in detached
+             * HEAD state).
+             *
+             * In the case of a plain repository, we can read the HEAD file directly, resolved directly from the .git directory.
+             *
+             * In the case of a worktree, we read the gitdir from the plain text .git file. This resolves to a directory from which we read
+             * the HEAD file and resolve commondir to the plain git repository.
+             */
+            final Path dotGit = project.getRootProject().getRootDir().toPath().resolve(".git");
+            final String revision;
+            if (Files.exists(dotGit) == false) {
+                return "unknown";
+            }
+            final Path head;
+            final Path gitDir;
+            if (Files.isDirectory(dotGit)) {
+                // this is a git repository, we can read HEAD directly
+                head = dotGit.resolve("HEAD");
+                gitDir = dotGit;
+            } else {
+                // this is a git worktree, follow the pointer to the repository
+                final Path workTree = Paths.get(readFirstLine(dotGit).substring("gitdir:".length()).trim());
+                head = workTree.resolve("HEAD");
+                final Path commonDir = Paths.get(readFirstLine(workTree.resolve("commondir")));
+                if (commonDir.isAbsolute()) {
+                    gitDir = commonDir;
+                } else {
+                    // this is the common case
+                    gitDir = workTree.resolve(commonDir);
                 }
-                return gitRevision.get();
             }
-        };
+            final String ref = readFirstLine(head);
+            if (ref.startsWith("ref:")) {
+                revision = readFirstLine(gitDir.resolve(ref.substring("ref:".length()).trim()));
+            } else {
+                // we are in detached HEAD state
+                revision = ref;
+            }
+            return revision;
+        } catch (final IOException e) {
+            // for now, do not be lenient until we have better understanding of real-world scenarios where this happens
+            throw new GradleException("unable to read the git revision", e);
+        }
+    }
 
+    private String readFirstLine(final Path path) throws IOException {
+        return Files.lines(path, StandardCharsets.UTF_8)
+            .findFirst()
+            .orElseThrow(() -> new IOException("file [" + path + "] is empty"));
     }
 
 }

client/rest-high-level/build.gradle

@@ -49,12 +49,14 @@ idea {
 dependencies {
   compile project(':server')
   compile project(':client:rest')
+  compile project(':modules:mapper-extras')
   compile project(':modules:parent-join')
   compile project(':modules:aggs-matrix-stats')
   compile project(':modules:rank-eval')
   compile project(':modules:lang-mustache')
   shadow project(':server')
   shadow project(':client:rest')
+  bundle project(':modules:mapper-extras')
   bundle project(':modules:parent-join')
   bundle project(':modules:aggs-matrix-stats')
   bundle project(':modules:rank-eval')

client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java

@@ -39,6 +39,7 @@
 import org.elasticsearch.client.ml.DeleteForecastRequest;
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
+import org.elasticsearch.client.ml.EstimateMemoryUsageRequest;
 import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
 import org.elasticsearch.client.ml.FindFileStructureRequest;
 import org.elasticsearch.client.ml.FlushJobRequest;
@@ -700,6 +701,15 @@ static Request evaluateDataFrame(EvaluateDataFrameRequest evaluateRequest) throw
         return request;
     }
 
+    static Request estimateMemoryUsage(EstimateMemoryUsageRequest estimateRequest) throws IOException {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "data_frame", "analytics", "_estimate_memory_usage")
+            .build();
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint);
+        request.setEntity(createEntity(estimateRequest, REQUEST_BODY_CONTENT_TYPE));
+        return request;
+    }
+
     static Request putFilter(PutFilterRequest putFilterRequest) throws IOException {
         String endpoint = new EndpointBuilder()
             .addPathPartAsIs("_ml")

client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java

@@ -34,6 +34,8 @@
 import org.elasticsearch.client.ml.DeleteJobRequest;
 import org.elasticsearch.client.ml.DeleteJobResponse;
 import org.elasticsearch.client.ml.DeleteModelSnapshotRequest;
+import org.elasticsearch.client.ml.EstimateMemoryUsageRequest;
+import org.elasticsearch.client.ml.EstimateMemoryUsageResponse;
 import org.elasticsearch.client.ml.EvaluateDataFrameRequest;
 import org.elasticsearch.client.ml.EvaluateDataFrameResponse;
 import org.elasticsearch.client.ml.FindFileStructureRequest;
@@ -2185,4 +2187,46 @@ public void evaluateDataFrameAsync(EvaluateDataFrameRequest request, RequestOpti
             listener,
             Collections.emptySet());
     }
+
+    /**
+     * Estimates memory usage for the given Data Frame Analytics
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/estimate-memory-usage-dfanalytics.html">
+     *     Estimate Memory Usage for Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link EstimateMemoryUsageRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return {@link EstimateMemoryUsageResponse} response object
+     * @throws IOException when there is a serialization issue sending the request or receiving the response
+     */
+    public EstimateMemoryUsageResponse estimateMemoryUsage(EstimateMemoryUsageRequest request,
+                                                           RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::estimateMemoryUsage,
+            options,
+            EstimateMemoryUsageResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Estimates memory usage for the given Data Frame Analytics asynchronously and notifies listener upon completion
+     * <p>
+     * For additional info
+     * see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/estimate-memory-usage-dfanalytics.html">
+     *     Estimate Memory Usage for Data Frame Analytics documentation</a>
+     *
+     * @param request The {@link EstimateMemoryUsageRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     */
+    public void estimateMemoryUsageAsync(EstimateMemoryUsageRequest request, RequestOptions options,
+                                         ActionListener<EstimateMemoryUsageResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::estimateMemoryUsage,
+            options,
+            EstimateMemoryUsageResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/core/IndexerJobStats.java

@@ -36,16 +36,16 @@ public abstract class IndexerJobStats {
     public static ParseField SEARCH_FAILURES = new ParseField("search_failures");
     public static ParseField INDEX_FAILURES = new ParseField("index_failures");
 
-    private final long numPages;
-    private final long numInputDocuments;
-    private final long numOuputDocuments;
-    private final long numInvocations;
-    private final long indexTime;
-    private final long indexTotal;
-    private final long searchTime;
-    private final long searchTotal;
-    private final long indexFailures;
-    private final long searchFailures;
+    protected final long numPages;
+    protected final long numInputDocuments;
+    protected final long numOuputDocuments;
+    protected final long numInvocations;
+    protected final long indexTime;
+    protected final long indexTotal;
+    protected final long searchTime;
+    protected final long searchTotal;
+    protected final long indexFailures;
+    protected final long searchFailures;
 
     public IndexerJobStats(long numPages, long numInputDocuments, long numOutputDocuments, long numInvocations,
                            long indexTime, long searchTime, long indexTotal, long searchTotal, long indexFailures, long searchFailures) {