Merge pull request #165 from elastic/master

🤖 ESQL: Merge upstream

Author: github-actions[bot]

.ci/bwcVersions

@@ -64,6 +64,6 @@ BWC_VERSION:
   - "8.2.1"
   - "8.2.2"
   - "8.2.3"
-  - "8.2.4"
   - "8.3.0"
+  - "8.3.1"
   - "8.4.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,4 @@
 BWC_VERSION:
   - "7.17.5"
-  - "8.2.4"
-  - "8.3.0"
+  - "8.3.1"
   - "8.4.0"

benchmarks/src/main/resources/org/elasticsearch/benchmark/xcontent/monitor_cluster_stats.json

@@ -298,16 +298,6 @@
           "extended_plugins" : [ ],
           "has_native_controller" : false
         },
-        {
-          "name" : "ingest-attachment",
-          "version" : "7.10.0",
-          "elasticsearch_version" : "7.10.0",
-          "java_version" : "1.8",
-          "description" : "Ingest processor that uses Apache Tika to extract contents",
-          "classname" : "org.elasticsearch.ingest.attachment.IngestAttachmentPlugin",
-          "extended_plugins" : [ ],
-          "has_native_controller" : false
-        },
         {
           "name" : "mapper-murmur3",
           "version" : "7.10.0",

build-tools-internal/version.properties

@@ -24,7 +24,7 @@ netty             = 4.1.77.Final
 commons_lang3                   = 3.9
 
 # when updating this version, you need to ensure compatibility with:
-#  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
+#  - modules/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
 #  - x-pack/plugin/security
 bouncycastle=1.64
@@ -36,9 +36,9 @@ opensaml = 4.0.1
 randomizedrunner  = 2.7.7
 junit             = 4.12
 junit5            = 5.7.1
-httpclient        = 4.5.10
-httpcore          = 4.4.12
-httpasyncclient   = 4.1.4
+httpclient        = 4.5.13
+httpcore          = 4.4.13
+httpasyncclient   = 4.1.5
 commonslogging    = 1.1.3
 commonscodec      = 1.14
 hamcrest          = 2.1

client/rest/licenses/httpasyncclient-4.1.4.jar.sha1

@@ -0,0 +1 @@
+cd18227f1eb8e9a263286c1d7362ceb24f6f9b32

client/rest/licenses/httpasyncclient-4.1.5.jar.sha1

@@ -0,0 +1 @@
+e5f6cae5ca7ecaac1ec2827a9e2d65ae2869cada

client/rest/licenses/httpclient-4.5.10.jar.sha1

@@ -0,0 +1 @@
+853b96d3afbb7bf8cc303fe27ee96836a10c1834

client/rest/licenses/httpclient-4.5.13.jar.sha1

@@ -0,0 +1 @@
+3f897ace4d7f10f0ea6a58f524a3b105dd483653

client/rest/licenses/httpcore-4.4.12.jar.sha1

@@ -0,0 +1 @@
+e5f6cae5ca7ecaac1ec2827a9e2d65ae2869cada

client/rest/licenses/httpcore-4.4.13.jar.sha1

@@ -0,0 +1 @@
+853b96d3afbb7bf8cc303fe27ee96836a10c1834

client/rest/licenses/httpcore-nio-4.4.12.jar.sha1

@@ -159,7 +159,12 @@ public class InstallPluginAction implements Closeable {
      * maintained so that existing user workflows that install these plugins do not need to be updated
      * immediately.
      */
-    public static final Set<String> PLUGINS_CONVERTED_TO_MODULES = Set.of("repository-azure", "repository-gcs", "repository-s3");
+    public static final Set<String> PLUGINS_CONVERTED_TO_MODULES = Set.of(
+        "repository-azure",
+        "repository-gcs",
+        "repository-s3",
+        "ingest-attachment"
+    );
 
     static final Set<PosixFilePermission> BIN_DIR_PERMS;
     static final Set<PosixFilePermission> BIN_FILES_PERMS;

client/rest/licenses/httpcore-nio-4.4.13.jar.sha1

@@ -1503,7 +1503,7 @@ public void testMultipleJars() throws Exception {
      * instead simply print a message to the terminal.
      */
     public void testInstallMigratedPlugins() throws Exception {
-        for (String id : List.of("repository-azure", "repository-gcs", "repository-s3")) {
+        for (String id : List.of("repository-azure", "repository-gcs", "repository-s3", "ingest-attachment")) {
             installPlugin(id);
             assertThat(terminal.getErrorOutput(), containsString("[" + id + "] is no longer a plugin"));
         }

client/sniffer/licenses/httpclient-4.5.10.jar.sha1

@@ -25,9 +25,9 @@ ext.docsFileTree = fileTree(projectDir) {
   // Broken code snippet tests
   exclude 'reference/graph/explore.asciidoc'
   if (BuildParams.inFipsJvm) {
-    // We don't install/support this plugin in FIPS 140
-    exclude 'plugins/ingest-attachment.asciidoc'
-    // We can't conditionally control output, this would be missing the ingest-attachment plugin
+    // We don't support this component in FIPS 140
+    exclude 'reference/ingest/processors/attachment.asciidoc'
+    // We can't conditionally control output, this would be missing the ingest-attachment component
     exclude 'reference/cat/plugins.asciidoc'
   }
 }
@@ -105,7 +105,7 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
       return
     }
     // Do not install ingest-attachment in a FIPS 140 JVM as this is not supported
-    if (subproj.path.startsWith(':plugins:ingest-attachment') && BuildParams.inFipsJvm) {
+    if (subproj.path.startsWith(':modules:ingest-attachment') && BuildParams.inFipsJvm) {
       return
     }
     plugin subproj.path

client/sniffer/licenses/httpclient-4.5.13.jar.sha1

@@ -0,0 +1,5 @@
+pr: 87491
+summary: Update version of internal http client
+area: "Infra/Core"
+type: enhancement
+issues: []

client/sniffer/licenses/httpcore-4.4.12.jar.sha1

@@ -0,0 +1,5 @@
+pr: 87895
+summary: Add processors to autoscaling capacity response
+area: Autoscaling
+type: enhancement
+issues: []

client/sniffer/licenses/httpcore-4.4.13.jar.sha1

@@ -0,0 +1,5 @@
+pr: 87978
+summary: Improve trained model stats API performance
+area: Machine Learning
+type: bug
+issues: []

distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/cli/InstallPluginAction.java

@@ -0,0 +1,5 @@
+pr: 87989
+summary: Move the ingest attachment processor to the default distribution
+area: Ingest
+type: enhancement
+issues: []

distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/cli/InstallPluginActionTests.java

@@ -0,0 +1,5 @@
+pr: 88031
+summary: Removing `BouncyCastle` dependencies from ingest-attachment plugin
+area: Ingest
+type: enhancement
+issues: []

docs/build.gradle

@@ -0,0 +1,5 @@
+pr: 88039
+summary: Update HDFS Repository to HDFS 3.3.3
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/87052.yaml

@@ -44,8 +44,6 @@ include::analysis.asciidoc[]
 
 include::discovery.asciidoc[]
 
-include::ingest.asciidoc[]
-
 include::mapper.asciidoc[]
 
 include::repository.asciidoc[]

docs/changelog/87163.yaml

@@ -145,3 +145,17 @@ include::redirects.asciidoc[tag=gcs-repo-migration]
 === Google Cloud Storage repository plugin
 
 include::redirects.asciidoc[tag=gcs-repo-migration]
+
+[role="exclude",id="ingest-attachment"]
+=== Ingest Attachment plugin
+
+// tag::ingest-attachment-migration[]
+The Ingest Attachment plugin is now included in {es}.
+See the {ref}/attachment.html[Ingest Attachment] processor.
+// end::ingest-attachment-migration[]
+
+[role="exclude",id="ingest"]
+=== Ingest plugins
+
+All ingest functionality has been moved into the default distribution.
+For more information refer to the {ref}/processors.html[existing processors].

docs/changelog/87235.yaml

@@ -7,8 +7,8 @@
 
 [IMPORTANT]
 ====
-cat APIs are only intended for human consumption using the command line or {kib} 
-console. They are _not_ intended for use by applications. For application 
+cat APIs are only intended for human consumption using the command line or {kib}
+console. They are _not_ intended for use by applications. For application
 consumption, use the <<cluster-nodes-info,nodes info API>>.
 ====
 
@@ -67,7 +67,6 @@ U7321H6 analysis-ukrainian      {version_qualified} The Ukrainian Analysis plugi
 U7321H6 discovery-azure-classic {version_qualified} The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism
 U7321H6 discovery-ec2           {version_qualified} The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism.
 U7321H6 discovery-gce           {version_qualified} The Google Compute Engine (GCE) Discovery plugin allows to use GCE API for the unicast discovery mechanism.
-U7321H6 ingest-attachment       {version_qualified} Ingest processor that uses Apache Tika to extract contents
 U7321H6 mapper-annotated-text   {version_qualified} The Mapper Annotated_text plugin adds support for text fields with markup used to inject annotation tokens into the index.
 U7321H6 mapper-murmur3          {version_qualified} The Mapper Murmur3 plugin allows to compute hashes of a field's values at index-time and to store them in the index.
 U7321H6 mapper-size             {version_qualified} The Mapper Size plugin allows document to record their uncompressed size at index time.

docs/changelog/87254.yaml

@@ -290,7 +290,8 @@ details have contents and a structure that is unique to each indicator.
     (Optional, array) A list of nodes that have been elected or replaced as master in a recent
     time window. This field is present if the master
     is changing rapidly enough to cause problems, and also present as additional information
-    when the indicator is `green`.
+    when the indicator is `green`. This array includes only elected masters, and does _not_
+    include empty entries for periods when there was no elected master.
 +
 .Properties of `recent_masters`
 [%collapsible%open]
@@ -303,17 +304,32 @@ details have contents and a structure that is unique to each indicator.
 ====
 
 `exception_fetching_history`::
-    (Optional, object) Master history is requested from the most recently elected master node for diagnosis purposes.
-    If fetching this remote history fails, the exception information is returned in this detail field.
+    (Optional, object) If the node being queried sees that the elected master has stepped down
+    repeatedly, the master history is requested from the most recently elected master node for
+    diagnosis purposes. If fetching this remote history fails, the exception information is
+    returned in this detail field.
 +
 .Properties of `exception_fetching_history`
 [%collapsible%open]
 ====
 `message`::
-(string) The exception message for the failed history fetch operation.
+    (string) The exception message for the failed history fetch operation.
 
 `stack_trace`::
-(string) The stack trace for the failed history fetch operation.
+    (string) The stack trace for the failed history fetch operation.
+====
+
+`cluster_formation`::
+    (Optional, object) If there has been no elected master node recently, the node being queried attempts to
+    gather information about why the cluster has been unable to form, or why the node being queried has been
+    unable to join the cluster if it has formed.
++
+.Properties of `cluster_formation`
+[%collapsible%open]
+====
+`description`::
+    (string) A detailed description explaining what went wrong with cluster formation, or why this node was
+    unable to join the cluster if it has formed.
 ====
 
 [[health-api-response-details-shards-availability]]

docs/changelog/87264.yaml

@@ -29,10 +29,11 @@ installed.
 
 [source,yaml]
 ----
-plugin.mandatory: ingest-attachment
+plugin.mandatory: my-ingest-plugin
 ----
 
 include::processors/append.asciidoc[]
+include::processors/attachment.asciidoc[]
 include::processors/bytes.asciidoc[]
 include::processors/circle.asciidoc[]
 include::processors/community-id.asciidoc[]