Scripting: Replace advanced and native scripts with ScriptEngine docs (#24603)

This commit documents how to write a `ScriptEngine` in order to use
expert internal apis, such as using Lucene directly to find index term
statistics. These documents prepare the way to remove both native
scripts and IndexLookup.

The example java code is actually compiled and tested under a new gradle
subproject for example plugins. This change does not yet breakup
jvm-example into the new examples dir, which should be done separately.

relates #19359
relates #19966

Author: rjernst

core/src/main/java/org/elasticsearch/script/LeafSearchScript.java

@@ -19,18 +19,30 @@
 
 package org.elasticsearch.script;
 
+import org.apache.lucene.search.Scorer;
 import org.elasticsearch.common.lucene.ScorerAware;
 
 import java.util.Map;
 
 /**
  * A per-segment {@link SearchScript}.
+ *
+ * This is effectively a functional interface, requiring at least implementing {@link #runAsDouble()}.
  */
 public interface LeafSearchScript extends ScorerAware, ExecutableScript {
 
-    void setDocument(int doc);
+    /**
+     * Set the document this script will process next.
+     */
+    default void setDocument(int doc) {}
+
+    @Override
+    default void setScorer(Scorer scorer) {}
 
-    void setSource(Map<String, Object> source);
+    /**
+     * Set the source for the current document.
+     */
+    default void setSource(Map<String, Object> source) {}
 
     /**
      * Sets per-document aggregation {@code _value}.
@@ -44,8 +56,23 @@ default void setNextAggregationValue(Object value) {
         setNextVar("_value", value);
     }
 
-    long runAsLong();
+    @Override
+    default void setNextVar(String field, Object value) {}
 
-    double runAsDouble();
+    /**
+     * Return the result as a long. This is used by aggregation scripts over long fields.
+     */
+    default long runAsLong() {
+        throw new UnsupportedOperationException("runAsLong is not implemented");
+    }
+
+    @Override
+    default Object run() {
+        return runAsDouble();
+    }
 
+    /**
+     * Return the result as a double. This is the main use case of search script, used for document scoring.
+     */
+    double runAsDouble();
 }

core/src/main/java/org/elasticsearch/script/ScriptEngineService.java

@@ -32,7 +32,12 @@ public interface ScriptEngineService extends Closeable {
 
     String getType();
 
-    String getExtension();
+    /**
+     * The extension for file scripts in this language.
+     */
+    default String getExtension() {
+        return getType();
+    }
 
     /**
      * Compiles a script.

docs/reference/modules/scripting.asciidoc

@@ -63,7 +63,7 @@ certain tasks.
     |built-in
     |templates
 
-|<<modules-scripting-native, `java`>>
+|<<modules-scripting-engine, `java`>>
     |n/a
     |you write it!
     |expert API
@@ -97,7 +97,4 @@ include::scripting/painless-debugging.asciidoc[]
 
 include::scripting/expression.asciidoc[]
 
-include::scripting/native.asciidoc[]
-
-include::scripting/advanced-scripting.asciidoc[]
-
+include::scripting/engine.asciidoc[]

docs/reference/modules/scripting/advanced-scripting.asciidoc

@@ -0,0 +1,57 @@
+[[modules-scripting-engine]]
+=== Advanced scripts using script engines
+
+A `ScriptEngine` is a backend for implementing a scripting language. It may also
+be used to write scripts that need to use advanced internals of scripting. For example,
+a script that wants to use term frequencies while scoring.
+
+The plugin {plugins}/plugin-authors.html[documentation] has more information on
+how to write a plugin so that Elasticsearch will properly load it. To register
+the `ScriptEngine`, your plugin should implement the `ScriptPlugin` interface
+and override the `getScriptEngine(Settings settings)` method.
+
+The following is an example of a custom `ScriptEngine` which uses the language
+name `expert_scripts`. It implements a single script called `pure_df` which
+may be used as a search script to override each document's score as
+the document frequency of a provided term.
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{docdir}/../../plugins/examples/script-expert-scoring/src/main/java/org/elasticsearch/example/expertscript/ExpertScriptPlugin.java[expert_engine]
+--------------------------------------------------
+
+You can execute the script by specifying its `lang` as `expert_scripts`, and the name
+of the script as the the script source:
+
+
+[source,js]
+--------------------------------------------------
+POST /_search
+{
+  "query": {
+    "function_score": {
+      "query": {
+        "match": {
+          "body": "foo"
+        }
+      },
+      "functions": [
+        {
+          "script_score": {
+            "script": {
+                "inline": "pure_df",
+                "lang" : "expert_scripts",
+                "params": {
+                    "field": "body",
+                    "term": "foo"
+                }
+            }
+          }
+        }
+      ]
+    }
+  }
+}
+--------------------------------------------------
+// CONSOLE
+// TEST[skip:we don't have an expert script plugin installed to test this]