Upgrade to latest Lucene snapshot (#33505)

* LeafCollector.setScorer() now takes a Scorable
* Scorers may not have null Weights
* IndexWriter.getFlushingBytes() reports how much memory is being used by IW threads writing to disk

Author: romseygeek

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 8.0.0-snapshot-4d78db26be
+lucene            = 8.0.0-snapshot-66c671ea80
 
 # optional dependencies
 spatial4j         = 0.7

modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+58b9db095c569b4c4da491810f14e1429878b594

modules/lang-expression/licenses/lucene-expressions-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -26,7 +26,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
-import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
@@ -336,7 +336,7 @@ public void setDocument(int docid) {
                     }
 
                     @Override
-                    public void setScorer(Scorer scorer) {
+                    public void setScorer(Scorable scorer) {
                         script.setScorer(scorer);
                     }
 

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java

@@ -19,39 +19,25 @@
 
 package org.elasticsearch.painless;
 
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Scorable;
 
-import java.io.IOException;
 import java.util.Collections;
 
 public class ScoreTests extends ScriptTestCase {
 
     /** Most of a dummy scorer impl that requires overriding just score(). */
-    abstract class MockScorer extends Scorer {
-        MockScorer() {
-            super(null);
-        }
+    abstract class MockScorer extends Scorable {
         @Override
         public int docID() {
             return 0;
         }
-        @Override
-        public DocIdSetIterator iterator() {
-            throw new UnsupportedOperationException();
-        }
     }
 
     public void testScoreWorks() {
         assertEquals(2.5, exec("_score", Collections.emptyMap(), Collections.emptyMap(),
             new MockScorer() {
                 @Override
-                public float score() throws IOException {
-                    return 2.5f;
-                }
-
-                @Override
-                public float getMaxScore(int upTo) throws IOException {
+                public float score() {
                     return 2.5f;
                 }
             },
@@ -62,14 +48,9 @@ public void testScoreNotUsed() {
         assertEquals(3.5, exec("3.5", Collections.emptyMap(), Collections.emptyMap(),
             new MockScorer() {
                 @Override
-                public float score() throws IOException {
+                public float score() {
                     throw new AssertionError("score() should not be called");
                 }
-
-                @Override
-                public float getMaxScore(int upTo) throws IOException {
-                    return Float.MAX_VALUE;
-                }
             },
             true));
     }
@@ -79,17 +60,12 @@ public void testScoreCached() {
             new MockScorer() {
                 private boolean used = false;
                 @Override
-                public float score() throws IOException {
+                public float score() {
                     if (used == false) {
                         return 4.5f;
                     }
                     throw new AssertionError("score() should not be called twice");
                 }
-
-                @Override
-                public float getMaxScore(int upTo) throws IOException {
-                    return 4.5f;
-                }
             },
             true));
     }

modules/lang-painless/src/test/java/org/elasticsearch/painless/ScoreTests.java

@@ -20,7 +20,7 @@
 package org.elasticsearch.painless;
 
 import junit.framework.AssertionFailedError;
-import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Scorable;
 import org.elasticsearch.common.lucene.ScorerAware;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.painless.antlr.Walker;
@@ -91,7 +91,7 @@ public Object exec(String script, Map<String, Object> vars, boolean picky) {
     }
 
     /** Compiles and returns the result of {@code script} with access to {@code vars} and compile-time parameters */
-    public Object exec(String script, Map<String, Object> vars, Map<String,String> compileParams, Scorer scorer, boolean picky) {
+    public Object exec(String script, Map<String, Object> vars, Map<String,String> compileParams, Scorable scorer, boolean picky) {
         // test for ambiguity errors before running the actual script if picky is true
         if (picky) {
             ScriptClassInfo scriptClassInfo = new ScriptClassInfo(PAINLESS_LOOKUP, GenericElasticsearchScript.class);

modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java

@@ -19,13 +19,11 @@
 
 package org.elasticsearch.painless;
 
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.Scorable;
 import org.elasticsearch.painless.spi.Whitelist;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptedMetricAggContexts;
 
-import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -66,20 +64,12 @@ public void testMapBasic() {
         Map<String, Object> params = new HashMap<>();
         Map<String, Object> state = new HashMap<>();
 
-        Scorer scorer = new Scorer(null) {
+        Scorable scorer = new Scorable() {
             @Override
             public int docID() { return 0; }
 
             @Override
             public float score() { return 0.5f; }
-
-            @Override
-            public DocIdSetIterator iterator() { return null; }
-
-            @Override
-            public float getMaxScore(int upTo) throws IOException {
-                return 0.5f;
-            }
         };
 
         ScriptedMetricAggContexts.MapScript.LeafFactory leafFactory = factory.newFactory(params, state, null);

modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptedMetricAggContextsTests.java

@@ -21,9 +21,9 @@
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedSetDocValues;
-import org.apache.lucene.search.ConstantScoreScorer;
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
@@ -148,7 +148,17 @@ protected void doPostCollection() throws IOException {
 
             final SortedSetDocValues globalOrdinals = valuesSource.globalOrdinalsValues(ctx);
             // Set the scorer, since we now replay only the child docIds
-            sub.setScorer(new ConstantScoreScorer(null, 1f, childDocsIter));
+            sub.setScorer(new Scorable() {
+                @Override
+                public float score() {
+                    return 1f;
+                }
+
+                @Override
+                public int docID() {
+                    return childDocsIter.docID();
+                }
+            });
 
             final Bits liveDocs = ctx.reader().getLiveDocs();
             for (int docId = childDocsIter

modules/parent-join/src/main/java/org/elasticsearch/join/aggregations/ParentToChildrenAggregator.java

@@ -0,0 +1 @@
+f009ee188453aabae77fad55aea08bc60323bb3e

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+af3d2ae975e3560c1ea69222d6c46072857952ba

plugins/analysis-icu/licenses/lucene-analyzers-icu-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+f17bc5e532d9dc2786a13bd577df64023d1baae1

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+7ad89d33c1cd960c91afa05b22024137fe108567

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+3f11fb254256d74e911b953994b47e7a95915954

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+b2348d140ef0c3e674cb81173f61c5e5f430facb

plugins/analysis-nori/licenses/lucene-analyzers-nori-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+485a0c3be58a5942b4a28639f1019181ef4cd0e3

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+a22f1c6749ca4a3fbc9b330161a8ea3301cac8de

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+41ce415b93d75662cc2e790d09120bc0234d6b1b

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+06c1e4fa838807059d27aaf5405cfdfe7303369c

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+5b0a019a938deb58160647e7640b348bb99c10a8

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+4d813f3ba0ddd56bac728edb88ed8875e6acfd18

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+00c7e20b6a35ebecc875dd52bfb324967c5555d6

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+e4dbff54a0befdc7d67c0f39890586c220df718e

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+74d17f6bdf1fa4d499f02904432aa3b1024bde88

server/licenses/lucene-analyzers-common-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+bec78be38f777765146c35f65e247909563d6814

server/licenses/lucene-analyzers-common-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+74b76f8fed44400bc2a5d938ca2611a97b4d7a7c

server/licenses/lucene-backward-codecs-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+2f65fa728b3bc924db6538f4c3caf2fcd25451cf

server/licenses/lucene-backward-codecs-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+916a91f0cab2d3684707c59e9adca7b3030b2c66

server/licenses/lucene-core-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+eb3e630d6013e41838fb277943ce921f256f1c61

server/licenses/lucene-core-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -0,0 +1 @@
+fa10ff14eab2f579cff2f0fa33c9c7f3b24daf12

server/licenses/lucene-grouping-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -0,0 +1 @@
+3dd65ca6612b4f98530847b99ab348fd83055fdf

server/licenses/lucene-grouping-8.0.0-snapshot-66c671ea80.jar.sha1

@@ -19,9 +19,9 @@
 package org.apache.lucene.search.grouping;
 
 import org.apache.lucene.search.FieldDoc;
+import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Sort;
 import org.apache.lucene.search.SortField;
 import org.apache.lucene.search.TotalHits;
@@ -44,7 +44,7 @@ public final class CollapsingTopDocsCollector<T> extends FirstPassGroupingCollec
     protected final String collapseField;
 
     protected final Sort sort;
-    protected Scorer scorer;
+    protected Scorable scorer;
 
     private int totalHitCount;
 
@@ -102,7 +102,7 @@ public ScoreMode scoreMode() {
     }
 
     @Override
-    public void setScorer(Scorer scorer) throws IOException {
+    public void setScorer(Scorable scorer) throws IOException {
         super.setScorer(scorer);
         this.scorer = scorer;
     }

server/licenses/lucene-highlighter-8.0.0-snapshot-4d78db26be.jar.sha1

@@ -19,8 +19,8 @@
 
 package org.elasticsearch.action.search;
 
+import org.apache.lucene.search.Scorable;
 import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.SimpleCollector;
 
 import java.io.IOException;
@@ -30,12 +30,12 @@
  */
 public class MaxScoreCollector extends SimpleCollector {
 
-    private Scorer scorer;
+    private Scorable scorer;
     private float maxScore = Float.NEGATIVE_INFINITY;
     private boolean hasHits = false;
 
     @Override
-    public void setScorer(Scorer scorer) {
+    public void setScorer(Scorable scorer) {
         this.scorer = scorer;
     }