Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x:
  Move score script context from SearchScript to its own class (#30816)
  Fix bad version check writing Repository nodes (#30846)
  Modify state of VerifyRepositoryResponse for bwc (#30762)
  QA: Fix tribe tests when running default zip
  Use remote client in TransportFieldCapsAction (#30838)
  Mute IndexMasterFailoverIT.testMasterFailoverDuringIndexingWithMappingChanges
  Ensure that ip_range aggregations always return bucket keys. (#30701)
  Limit user to single concurrent auth per realm (#30794)
  Security: fix dynamic mapping updates with aliases (#30787)
  [Tests] Move templated _rank_eval tests (#30679)
  Move Watcher versioning setting to meta field (#30832)
  Restore "Add more yaml tests for get alias API " (#30814)
  Send client headers from TransportClient (#30803)
  [Docs] Explain incomplete dates in range queries (#30689)
  Move persistent task registrations to core (#30755)
  Decouple ClusterStateTaskListener & ClusterApplier (#30809)
  Packaging: Ensure upgrade_is_oss flag file is always deleted (#30732)
  Force stable file modes for built packages (#30823)

Author: martijnvg

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -11,6 +11,7 @@ import org.gradle.api.internal.artifacts.dependencies.DefaultProjectDependency
 import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Delete
 import org.gradle.api.tasks.Exec
+import org.gradle.api.tasks.StopExecutionException
 import org.gradle.api.tasks.TaskState
 
 import static java.util.Collections.unmodifiableList
@@ -285,8 +286,10 @@ class VagrantTestPlugin implements Plugin<Project> {
             dependsOn copyPackagingArchives
             doFirst {
                 project.delete("${archivesDir}/upgrade_is_oss")
+                if (project.extensions.esvagrant.upgradeFromVersion.before('6.3.0')) {
+                    throw new StopExecutionException("upgrade version is before 6.3.0")
+                }
             }
-            onlyIf { project.extensions.esvagrant.upgradeFromVersion.onOrAfter('6.3.0') }
             file "${archivesDir}/upgrade_is_oss"
             contents ''
         }

distribution/build.gradle

@@ -242,6 +242,8 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
           if (it.relativePath.segments[-2] == 'bin') {
             // bin files, wherever they are within modules (eg platform specific) should be executable
             it.mode = 0755
+          } else {
+            it.mode = 0644
           }
         }
         if (oss) {

distribution/packages/build.gradle

@@ -122,6 +122,7 @@ Closure commonPackageConfig(String type, boolean oss) {
       }
       from(rootProject.projectDir) {
         include 'README.textile'
+        fileMode 0644
       }
       into('modules') {
         with copySpec {
@@ -135,6 +136,11 @@ Closure commonPackageConfig(String type, boolean oss) {
             for (int i = segments.length - 2; i > 0 && segments[i] != 'modules'; --i) {
               directory('/' + segments[0..i].join('/'), 0755)
             }
+            if (segments[-2] == 'bin') {
+              fcp.mode = 0755
+            } else {
+              fcp.mode = 0644
+            }
           }
         }
       }
@@ -153,6 +159,7 @@ Closure commonPackageConfig(String type, boolean oss) {
           include oss ? 'APACHE-LICENSE-2.0.txt' : 'ELASTIC-LICENSE.txt'
           rename { 'LICENSE.txt' }
         }
+        fileMode 0644
       }
     }
 
@@ -181,14 +188,17 @@ Closure commonPackageConfig(String type, boolean oss) {
     configurationFile '/usr/lib/systemd/system/elasticsearch.service'
     into('/usr/lib/tmpfiles.d') {
       from "${packagingFiles}/systemd/elasticsearch.conf"
+      fileMode 0644
     }
     into('/usr/lib/systemd/system') {
       fileType CONFIG | NOREPLACE
       from "${packagingFiles}/systemd/elasticsearch.service"
+      fileMode 0644
     }
     into('/usr/lib/sysctl.d') {
       fileType CONFIG | NOREPLACE
       from "${packagingFiles}/systemd/sysctl/elasticsearch.conf"
+      fileMode 0644
     }
 
     // ========= sysV init =========

docs/reference/aggregations/bucket/iprange-aggregation.asciidoc

@@ -37,10 +37,12 @@ Response:
         "ip_ranges": {
             "buckets" : [
                 {
+                    "key": "*-10.0.0.5",
                     "to": "10.0.0.5",
                     "doc_count": 10
                 },
                 {
+                    "key": "10.0.0.5-*",
                     "from": "10.0.0.5",
                     "doc_count": 260
                 }

docs/reference/query-dsl/range-query.asciidoc

@@ -109,6 +109,12 @@ GET _search
 --------------------------------------------------
 // CONSOLE 
 
+Note that if the date misses some of the year, month and day coordinates, the
+missing parts are filled with the start of
+https://en.wikipedia.org/wiki/Unix_time[unix time], which is January 1st, 1970.
+This means, that when e.g. specifying `dd` as the format, a value like `"gte" : 10`
+will translate to `1970-01-10T00:00:00.000Z`.
+
 ===== Time zone in range queries
 
 Dates can be converted from another timezone to UTC either by specifying the

modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java

@@ -23,8 +23,10 @@
 import org.apache.lucene.expressions.SimpleBindings;
 import org.apache.lucene.expressions.js.JavascriptCompiler;
 import org.apache.lucene.expressions.js.VariableContext;
+import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.queries.function.ValueSource;
 import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource;
+import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.SortField;
 import org.elasticsearch.SpecialPermission;
 import org.elasticsearch.common.Nullable;
@@ -39,12 +41,14 @@
 import org.elasticsearch.script.ClassPermission;
 import org.elasticsearch.script.ExecutableScript;
 import org.elasticsearch.script.FilterScript;
+import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptEngine;
 import org.elasticsearch.script.ScriptException;
 import org.elasticsearch.script.SearchScript;
 import org.elasticsearch.search.lookup.SearchLookup;
 
+import java.io.IOException;
 import java.security.AccessControlContext;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
@@ -111,6 +115,9 @@ protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundE
         } else if (context.instanceClazz.equals(FilterScript.class)) {
             FilterScript.Factory factory = (p, lookup) -> newFilterScript(expr, lookup, p);
             return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(ScoreScript.class)) {
+            ScoreScript.Factory factory = (p, lookup) -> newScoreScript(expr, lookup, p);
+            return context.factoryClazz.cast(factory);
         }
         throw new IllegalArgumentException("expression engine does not know how to handle script context [" + context.name + "]");
     }
@@ -260,6 +267,42 @@ public void setDocument(int docid) {
             };
         };
     }
+    
+    private ScoreScript.LeafFactory newScoreScript(Expression expr, SearchLookup lookup, @Nullable Map<String, Object> vars) {
+        SearchScript.LeafFactory searchLeafFactory = newSearchScript(expr, lookup, vars);
+        return new ScoreScript.LeafFactory() {
+            @Override
+            public boolean needs_score() {
+                return searchLeafFactory.needs_score();
+            }
+
+            @Override
+            public ScoreScript newInstance(LeafReaderContext ctx) throws IOException {
+                SearchScript script = searchLeafFactory.newInstance(ctx);
+                return new ScoreScript(vars, lookup, ctx) {
+                    @Override
+                    public double execute() {
+                        return script.runAsDouble();
+                    }
+    
+                    @Override
+                    public void setDocument(int docid) {
+                        script.setDocument(docid);
+                    }
+    
+                    @Override
+                    public void setScorer(Scorer scorer) {
+                        script.setScorer(scorer);
+                    }
+    
+                    @Override
+                    public double get_score() {
+                        return script.getScore();
+                    }
+                };
+            }
+        };
+    }
 
     /**
      * converts a ParseException at compile-time or link-time to a ScriptException

plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java

@@ -30,9 +30,9 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ScriptPlugin;
+import org.elasticsearch.script.ScoreScript;
 import org.elasticsearch.script.ScriptContext;
 import org.elasticsearch.script.ScriptEngine;
-import org.elasticsearch.script.SearchScript;
 
 /**
  * An example script plugin that adds a {@link ScriptEngine} implementing expert scoring.
@@ -54,12 +54,12 @@ public String getType() {
 
         @Override
         public <T> T compile(String scriptName, String scriptSource, ScriptContext<T> context, Map<String, String> params) {
-            if (context.equals(SearchScript.SCRIPT_SCORE_CONTEXT) == false) {
+            if (context.equals(ScoreScript.CONTEXT) == false) {
                 throw new IllegalArgumentException(getType() + " scripts cannot be used for context [" + context.name + "]");
             }
             // we use the script "source" as the script identifier
             if ("pure_df".equals(scriptSource)) {
-                SearchScript.Factory factory = (p, lookup) -> new SearchScript.LeafFactory() {
+                ScoreScript.Factory factory = (p, lookup) -> new ScoreScript.LeafFactory() {
                     final String field;
                     final String term;
                     {
@@ -74,18 +74,18 @@ public <T> T compile(String scriptName, String scriptSource, ScriptContext<T> co
                     }
 
                     @Override
-                    public SearchScript newInstance(LeafReaderContext context) throws IOException {
+                    public ScoreScript newInstance(LeafReaderContext context) throws IOException {
                         PostingsEnum postings = context.reader().postings(new Term(field, term));
                         if (postings == null) {
                             // the field and/or term don't exist in this segment, so always return 0
-                            return new SearchScript(p, lookup, context) {
+                            return new ScoreScript(p, lookup, context) {
                                 @Override
-                                public double runAsDouble() {
+                                public double execute() {
                                     return 0.0d;
                                 }
                             };
                         }
-                        return new SearchScript(p, lookup, context) {
+                        return new ScoreScript(p, lookup, context) {
                             int currentDocid = -1;
                             @Override
                             public void setDocument(int docid) {
@@ -100,7 +100,7 @@ public void setDocument(int docid) {
                                 currentDocid = docid;
                             }
                             @Override
-                            public double runAsDouble() {
+                            public double execute() {
                                 if (postings.docID() != currentDocid) {
                                     // advance moved past the current doc, so this doc has no occurrences of the term
                                     return 0.0d;

qa/smoke-test-rank-eval-with-mustache/build.gradle

@@ -31,6 +31,3 @@ dependencies {
  * and will be fixed later.
  * Tracked by https://github.com/elastic/elasticsearch/issues/30628
  */
-if ("zip".equals(integTestCluster.distribution)) {
-    integTestRunner.enabled = false
-}