Merge branch 'master' into compile-with-jdk-9

* master:
  test: replaced try-catch statements with expectThrows(...)
  Add getWarmer and getTranslog method to NodeIndicesStats (#28092)
  fix doc mistake
  Added ASN support for Ingest GeoIP plugin.
  Fix global aggregation that requires breadth first and scores (#27942)
  Introduce Gradle wrapper
  Ignore GIT_COMMIT when calculating commit hash
  Re-enable bwc tests after #27881 was backported

Author: jasontedor

.gitignore

@@ -20,11 +20,6 @@ nbactions.xml
 .gradle/
 build/
 
-# gradle wrapper 
-/gradle/
-gradlew
-gradlew.bat
-
 # maven stuff (to be removed when trunk becomes 4.x)
 *-execution-hints.log
 target/

CONTRIBUTING.md

@@ -99,9 +99,8 @@ However, since Elasticsearch, supports JDK 8 the build supports compiling with
 JDK 9 and testing on a JDK 8 runtime; to do this, set `JAVA_8_HOME` pointing to
 the Java home of a JDK 8 installation. 
 
-Make sure you have [Gradle](http://gradle.org) installed, as
-Elasticsearch uses it as its build system. Gradle must be at least
-version 4.3 in order to build successfully.
+Elasticsearch uses the Gradle wrapper for its build. You can execute Gradle
+using the wrapper via the `gradlew` script in the root of the repository.
 
 We support development in the Eclipse and IntelliJ IDEs. For Eclipse, the
 minimum version that we support is [Eclipse Oxygen][eclipse] (version 4.7). For

build.gradle

@@ -25,7 +25,13 @@ import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
+import org.gradle.api.tasks.wrapper.Wrapper
+import org.gradle.api.tasks.wrapper.Wrapper.DistributionType
+import org.gradle.util.GradleVersion
+import org.gradle.util.DistributionLocator
+
 import java.nio.file.Path
+import java.security.MessageDigest
 
 // common maven publishing configuration
 subprojects {
@@ -139,8 +145,7 @@ task verifyVersions {
  * after the backport of the backcompat code is complete.
  */
 allprojects {
-  // TODO: re-enable after https://github.com/elastic/elasticsearch/pull/27881 is backported
-  ext.bwc_tests_enabled = false
+  ext.bwc_tests_enabled = true
 }
 
 task verifyBwcTestsEnabled {
@@ -403,6 +408,27 @@ task run(type: Run) {
   impliesSubProjects = true
 }
 
+task wrapper(type: Wrapper)
+
+gradle.projectsEvaluated {
+
+  allprojects {
+    tasks.withType(Wrapper) { Wrapper wrapper ->
+      wrapper.distributionType = DistributionType.ALL
+
+      wrapper.doLast {
+        final DistributionLocator locator = new DistributionLocator()
+        final GradleVersion version = GradleVersion.version(wrapper.gradleVersion)
+        final URI distributionUri = locator.getDistributionFor(version, wrapper.distributionType.name().toLowerCase(Locale.ENGLISH))
+        final URI sha256Uri = new URI(distributionUri.toString() + ".sha256")
+        final String sha256Sum = new String(sha256Uri.toURL().bytes)
+        wrapper.getPropertiesFile() << "distributionSha256Sum=${sha256Sum}\n"
+      }
+    }
+  }
+
+}
+
 /* Remove assemble on all qa projects because we don't need to publish
  * artifacts for them. */
 gradle.projectsEvaluated {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -21,6 +21,8 @@ package org.elasticsearch.gradle
 import com.carrotsearch.gradle.junit4.RandomizedTestingTask
 import nebula.plugin.extraconfigurations.ProvidedBasePlugin
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.eclipse.jgit.lib.Constants
+import org.eclipse.jgit.lib.RepositoryBuilder
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.GradleException
 import org.gradle.api.InvalidUserDataException
@@ -529,6 +531,17 @@ class BuildPlugin implements Plugin<Project> {
                 if (jarTask.manifest.attributes.containsKey('Change') == false) {
                     logger.warn('Building without git revision id.')
                     jarTask.manifest.attributes('Change': 'Unknown')
+                } else {
+                    /*
+                     * The info-scm plugin assumes that if GIT_COMMIT is set it was set by Jenkins to the commit hash for this build.
+                     * However, that assumption is wrong as this build could be a sub-build of another Jenkins build for which GIT_COMMIT
+                     * is the commit hash for that build. Therefore, if GIT_COMMIT is set we calculate the commit hash ourselves.
+                     */
+                    if (System.getenv("GIT_COMMIT") != null) {
+                        final String hash = new RepositoryBuilder().findGitDir(project.buildDir).build().resolve(Constants.HEAD).name
+                        final String shortHash = hash?.substring(0, 7)
+                        jarTask.manifest.attributes('Change': shortHash)
+                    }
                 }
             }
             // add license/notice files

core/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java

@@ -26,7 +26,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Streamable;
-import org.elasticsearch.common.xcontent.ToXContent.Params;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.Index;
@@ -43,6 +42,8 @@
 import org.elasticsearch.index.shard.DocsStats;
 import org.elasticsearch.index.shard.IndexingStats;
 import org.elasticsearch.index.store.StoreStats;
+import org.elasticsearch.index.translog.TranslogStats;
+import org.elasticsearch.index.warmer.WarmerStats;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 
 import java.io.IOException;
@@ -117,6 +118,11 @@ public FlushStats getFlush() {
         return stats.getFlush();
     }
 
+    @Nullable
+    public WarmerStats getWarmer() {
+        return stats.getWarmer();
+    }
+
     @Nullable
     public FieldDataStats getFieldData() {
         return stats.getFieldData();
@@ -142,6 +148,11 @@ public SegmentsStats getSegments() {
         return stats.getSegments();
     }
 
+    @Nullable
+    public TranslogStats getTranslog() {
+        return stats.getTranslog();
+    }
+
     @Nullable
     public RecoveryStats getRecoveryStats() {
         return stats.getRecoveryStats();

core/src/main/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollector.java

@@ -21,6 +21,8 @@
 
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.packed.PackedInts;
@@ -59,16 +61,22 @@ private static class Entry {
     final List<Entry> entries = new ArrayList<>();
     BucketCollector collector;
     final SearchContext searchContext;
+    final boolean isGlobal;
     LeafReaderContext context;
     PackedLongValues.Builder docDeltas;
     PackedLongValues.Builder buckets;
     long maxBucket = -1;
     boolean finished = false;
     LongHash selectedBuckets;
 
-    /** Sole constructor. */
-    public BestBucketsDeferringCollector(SearchContext context) {
+    /**
+     * Sole constructor.
+     * @param context The search context
+     * @param isGlobal Whether this collector visits all documents (global context)
+     */
+    public BestBucketsDeferringCollector(SearchContext context, boolean isGlobal) {
         this.searchContext = context;
+        this.isGlobal = isGlobal;
     }
 
     @Override
@@ -144,11 +152,11 @@ public void prepareSelectedBuckets(long... selectedBuckets) throws IOException {
         }
         this.selectedBuckets = hash;
 
-        boolean needsScores = collector.needsScores();
+        boolean needsScores = needsScores();
         Weight weight = null;
         if (needsScores) {
-            weight = searchContext.searcher()
-                        .createNormalizedWeight(searchContext.query(), true);
+            Query query = isGlobal ? new MatchAllDocsQuery() : searchContext.query();
+            weight = searchContext.searcher().createNormalizedWeight(query, true);
         }
         for (Entry entry : entries) {
             final LeafBucketCollector leafCollector = collector.getLeafCollector(entry.context);

core/src/main/java/org/elasticsearch/search/aggregations/bucket/DeferableBucketAggregator.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.BucketCollector;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregator;
 import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
 import org.elasticsearch.search.internal.SearchContext;
 
@@ -61,10 +62,20 @@ protected void doPreCollection() throws IOException {
         collectableSubAggregators = BucketCollector.wrap(collectors);
     }
 
+    public static boolean descendsFromGlobalAggregator(Aggregator parent) {
+        while (parent != null) {
+            if (parent.getClass() == GlobalAggregator.class) {
+                return true;
+            }
+            parent = parent.parent();
+        }
+        return false;
+    }
+
     public DeferringBucketCollector getDeferringCollector() {
         // Default impl is a collector that selects the best buckets
         // but an alternative defer policy may be based on best docs.
-        return new BestBucketsDeferringCollector(context());
+        return new BestBucketsDeferringCollector(context(), descendsFromGlobalAggregator(parent()));
     }
 
     /**
@@ -74,7 +85,7 @@ public DeferringBucketCollector getDeferringCollector() {
      * recording of all doc/bucketIds from the first pass and then the sub class
      * should call {@link #runDeferredCollections(long...)} for the selected set
      * of buckets that survive the pruning.
-     * 
+     *
      * @param aggregator
      *            the child aggregator
      * @return true if the aggregator should be deferred until a first pass at

core/src/test/java/org/elasticsearch/search/aggregations/bucket/BestBucketsDeferringCollectorTests.java

@@ -27,6 +27,8 @@
 import org.apache.lucene.index.RandomIndexWriter;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
@@ -41,6 +43,8 @@
 import java.util.HashSet;
 import java.util.Set;
 
+import static org.mockito.Mockito.when;
+
 public class BestBucketsDeferringCollectorTests extends AggregatorTestCase {
 
     public void testReplay() throws Exception {
@@ -59,17 +63,38 @@ public void testReplay() throws Exception {
         IndexSearcher indexSearcher = new IndexSearcher(indexReader);
 
         TermQuery termQuery = new TermQuery(new Term("field", String.valueOf(randomInt(maxNumValues))));
+        Query rewrittenQuery = indexSearcher.rewrite(termQuery);
         TopDocs topDocs = indexSearcher.search(termQuery, numDocs);
 
         SearchContext searchContext = createSearchContext(indexSearcher, createIndexSettings());
-        BestBucketsDeferringCollector collector = new BestBucketsDeferringCollector(searchContext);
+        when(searchContext.query()).thenReturn(rewrittenQuery);
+        BestBucketsDeferringCollector collector = new BestBucketsDeferringCollector(searchContext, false) {
+            @Override
+            public boolean needsScores() {
+                return true;
+            }
+        };
         Set<Integer> deferredCollectedDocIds = new HashSet<>();
         collector.setDeferredCollector(Collections.singleton(bla(deferredCollectedDocIds)));
         collector.preCollection();
         indexSearcher.search(termQuery, collector);
         collector.postCollection();
         collector.replay(0);
 
+        assertEquals(topDocs.scoreDocs.length, deferredCollectedDocIds.size());
+        for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
+            assertTrue("expected docid [" + scoreDoc.doc + "] is missing", deferredCollectedDocIds.contains(scoreDoc.doc));
+        }
+
+        topDocs = indexSearcher.search(new MatchAllDocsQuery(), numDocs);
+        collector = new BestBucketsDeferringCollector(searchContext, true);
+        deferredCollectedDocIds = new HashSet<>();
+        collector.setDeferredCollector(Collections.singleton(bla(deferredCollectedDocIds)));
+        collector.preCollection();
+        indexSearcher.search(new MatchAllDocsQuery(), collector);
+        collector.postCollection();
+        collector.replay(0);
+
         assertEquals(topDocs.scoreDocs.length, deferredCollectedDocIds.size());
         for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
             assertTrue("expected docid [" + scoreDoc.doc + "] is missing", deferredCollectedDocIds.contains(scoreDoc.doc));

core/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java

@@ -46,14 +46,21 @@
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.AggregationBuilder;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.BucketOrder;
 import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation;
+import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal;
+import org.elasticsearch.search.aggregations.metrics.tophits.InternalTopHits;
+import org.elasticsearch.search.aggregations.metrics.tophits.TopHitsAggregationBuilder;
 import org.elasticsearch.search.aggregations.support.ValueType;
 
 import java.io.IOException;
@@ -67,6 +74,8 @@
 import java.util.function.BiFunction;
 import java.util.function.Function;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.instanceOf;
 
 public class TermsAggregatorTests extends AggregatorTestCase {
@@ -933,6 +942,63 @@ public void testMixLongAndDouble() throws Exception {
         }
     }
 
+    public void testGlobalAggregationWithScore() throws IOException {
+        try (Directory directory = newDirectory()) {
+            try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                Document document = new Document();
+                document.add(new SortedDocValuesField("keyword", new BytesRef("a")));
+                indexWriter.addDocument(document);
+                document = new Document();
+                document.add(new SortedDocValuesField("keyword", new BytesRef("c")));
+                indexWriter.addDocument(document);
+                document = new Document();
+                document.add(new SortedDocValuesField("keyword", new BytesRef("e")));
+                indexWriter.addDocument(document);
+                try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) {
+                    IndexSearcher indexSearcher = newIndexSearcher(indexReader);
+                    String executionHint = randomFrom(TermsAggregatorFactory.ExecutionMode.values()).toString();
+                    Aggregator.SubAggCollectionMode collectionMode = randomFrom(Aggregator.SubAggCollectionMode.values());
+                    GlobalAggregationBuilder globalBuilder = new GlobalAggregationBuilder("global")
+                        .subAggregation(
+                            new TermsAggregationBuilder("terms", ValueType.STRING)
+                                .executionHint(executionHint)
+                                .collectMode(collectionMode)
+                                .field("keyword")
+                                .order(BucketOrder.key(true))
+                                .subAggregation(
+                                    new TermsAggregationBuilder("sub_terms", ValueType.STRING)
+                                        .executionHint(executionHint)
+                                        .collectMode(collectionMode)
+                                        .field("keyword").order(BucketOrder.key(true))
+                                        .subAggregation(
+                                            new TopHitsAggregationBuilder("top_hits")
+                                                .storedField("_none_")
+                                        )
+                                )
+                        );
+
+                    MappedFieldType fieldType = new KeywordFieldMapper.KeywordFieldType();
+                    fieldType.setName("keyword");
+                    fieldType.setHasDocValues(true);
+
+                    InternalGlobal result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), globalBuilder, fieldType);
+                    InternalMultiBucketAggregation<?, ?> terms = result.getAggregations().get("terms");
+                    assertThat(terms.getBuckets().size(), equalTo(3));
+                    for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) {
+                        InternalMultiBucketAggregation<?, ?> subTerms = bucket.getAggregations().get("sub_terms");
+                        assertThat(subTerms.getBuckets().size(), equalTo(1));
+                        MultiBucketsAggregation.Bucket subBucket  = subTerms.getBuckets().get(0);
+                        InternalTopHits topHits = subBucket.getAggregations().get("top_hits");
+                        assertThat(topHits.getHits().getHits().length, equalTo(1));
+                        for (SearchHit hit : topHits.getHits()) {
+                            assertThat(hit.getScore(), greaterThan(0f));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     private IndexReader createIndexWithLongs() throws IOException {
         Directory directory = newDirectory();
         RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);