Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

docs/changelog/120302.yaml

@@ -0,0 +1,6 @@
+pr: 120302
+summary: "ESQL: Enhanced `DATE_TRUNC` with arbitrary intervals"
+area: ES|QL
+type: enhancement
+issues:
+  - 120094

docs/changelog/121041.yaml

@@ -0,0 +1,5 @@
+pr: 121041
+summary: Support configurable chunking in `semantic_text` fields
+area: Relevance
+type: enhancement
+issues: []

docs/changelog/125060.yaml

@@ -0,0 +1,16 @@
+pr: 125060
+summary: Allow partial results by default in ES|QL
+area: ES|QL
+type: breaking
+issues: [122802]
+
+breaking:
+  title: Allow partial results by default in ES|QL
+  area: ES|QL
+  details: >-
+    In earlier versions of {es}, ES|QL would fail the entire query if it encountered any error. ES|QL now returns partial results instead of failing when encountering errors.
+
+  impact: >-
+    Callers should check the `is_partial` flag returned in the response to determine if the result is partial or complete. If returning partial results is not desired, this option can be overridden per request via an `allow_partial_results` parameter in the query URL or globally via the cluster setting `esql.query.allow_partial_results`.
+
+  notable: true

docs/changelog/125452.yaml

@@ -0,0 +1,5 @@
+pr: 125452
+summary: Add GCS telemetry with `ThreadLocal`
+area: Snapshot/Restore
+type: enhancement
+issues: []

docs/changelog/125732.yaml

@@ -0,0 +1,5 @@
+pr: 125732
+summary: Log stack traces on data nodes before they are cleared for transport
+area: Search
+type: bug
+issues: []

docs/changelog/125896.yaml

@@ -0,0 +1,5 @@
+pr: 125896
+summary: Support explicit Z/M attributes using WKT geometry
+area: Geo
+type: enhancement
+issues: [123111]

docs/changelog/126191.yaml

@@ -0,0 +1,5 @@
+pr: 126191
+summary: Fix NPE for missing Content Type header in OIDC Authenticator
+area: Authentication
+type: bug
+issues: []

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -14,6 +14,11 @@ To learn how to upgrade, check out <uprade docs>.
 
 % ## Next version [elasticsearch-nextversion-breaking-changes]
 
+## 9.1.0 [elasticsearch-910-breaking-changes]
+
+ES|QL
+:   * Allow partial results by default in ES|QL [#125060](https://github.com/elastic/elasticsearch/pull/125060)
+
 ## 9.0.0 [elasticsearch-900-breaking-changes]
 
 Allocation

docs/reference/query-languages/esql/_snippets/functions/description/date_trunc.md

@@ -141,7 +141,9 @@ private static void validatePolicyScopes(String layerName, Policy policy, Set<St
     public static Policy parsePolicyIfExists(String pluginName, Path pluginRoot, boolean isExternalPlugin) throws IOException {
         Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
         if (Files.exists(policyFile)) {
-            return new PolicyParser(Files.newInputStream(policyFile, StandardOpenOption.READ), pluginName, isExternalPlugin).parsePolicy();
+            try (var inputStream = Files.newInputStream(policyFile, StandardOpenOption.READ)) {
+                return new PolicyParser(inputStream, pluginName, isExternalPlugin).parsePolicy();
+            }
         }
         return new Policy(pluginName, List.of());
     }

docs/reference/query-languages/esql/_snippets/functions/examples/bucket.md

@@ -16,6 +16,7 @@
 import org.elasticsearch.entitlement.runtime.policy.Platform;
 import org.elasticsearch.entitlement.runtime.policy.PolicyValidationException;
 
+import java.nio.file.FileSystems;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -31,6 +32,8 @@
  */
 public record FilesEntitlement(List<FileData> filesData) implements Entitlement {
 
+    public static final String SEPARATOR = FileSystems.getDefault().getSeparator();
+
     public static final FilesEntitlement EMPTY = new FilesEntitlement(List.of());
 
     public enum Mode {
@@ -160,7 +163,7 @@ public FileData withPlatform(Platform platform) {
 
         @Override
         public String description() {
-            return Strings.format("[%s] <%s>/%s%s", mode, baseDir, relativePath, exclusive ? " (exclusive)" : "");
+            return Strings.format("[%s] <%s>%s%s%s", mode, baseDir, SEPARATOR, relativePath, exclusive ? " (exclusive)" : "");
         }
     }
 
@@ -192,7 +195,7 @@ public FileData withPlatform(Platform platform) {
 
         @Override
         public String description() {
-            return Strings.format("[%s] <%s>/<%s>%s", mode, baseDir, setting, exclusive ? " (exclusive)" : "");
+            return Strings.format("[%s] <%s>%s<%s>%s", mode, baseDir, SEPARATOR, setting, exclusive ? " (exclusive)" : "");
         }
     }
 

docs/reference/query-languages/esql/kibana/definition/functions/date_trunc.json

@@ -27,6 +27,7 @@
 import java.util.List;
 import java.util.Set;
 
+import static org.elasticsearch.entitlement.runtime.policy.entitlements.FilesEntitlement.SEPARATOR;
 import static org.elasticsearch.test.LambdaMatchers.transformedMatch;
 import static org.hamcrest.Matchers.both;
 import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -317,6 +318,7 @@ public void testFormatPolicyWithMultipleScopes() {
     /** Test that we can format some simple files entitlement properly */
     public void testFormatFilesEntitlement() {
         var pathAB = Path.of("/a/b");
+        var pathCD = Path.of("c/d");
         var policy = new Policy(
             "test-plugin",
             List.of(
@@ -326,11 +328,7 @@ public void testFormatFilesEntitlement() {
                         new FilesEntitlement(
                             List.of(
                                 FilesEntitlement.FileData.ofPath(pathAB, FilesEntitlement.Mode.READ_WRITE),
-                                FilesEntitlement.FileData.ofRelativePath(
-                                    Path.of("c/d"),
-                                    FilesEntitlement.BaseDir.DATA,
-                                    FilesEntitlement.Mode.READ
-                                )
+                                FilesEntitlement.FileData.ofRelativePath(pathCD, FilesEntitlement.BaseDir.DATA, FilesEntitlement.Mode.READ)
                             )
                         )
                     )
@@ -353,7 +351,17 @@ public void testFormatFilesEntitlement() {
             )
         );
         Set<String> actual = PolicyUtils.getEntitlementsDescriptions(policy);
-        assertThat(actual, containsInAnyOrder("files [READ_WRITE] " + pathAB, "files [READ] <DATA>/c/d", "files [READ] <DATA>/<setting>"));
+        var pathABString = pathAB.toAbsolutePath().toString();
+        var pathCDString = SEPARATOR + pathCD.toString();
+        var pathSettingString = SEPARATOR + "<setting>";
+        assertThat(
+            actual,
+            containsInAnyOrder(
+                "files [READ_WRITE] " + pathABString,
+                "files [READ] <DATA>" + pathCDString,
+                "files [READ] <DATA>" + pathSettingString
+            )
+        );
     }
 
     /** Test that we can format some simple files entitlement properly */

docs/reference/query-languages/esql/kibana/docs/functions/date_trunc.md

@@ -44,6 +44,8 @@ public class WellKnownText {
     public static final String RPAREN = ")";
     public static final String COMMA = ",";
     public static final String NAN = "NaN";
+    public static final String Z = "Z";
+    public static final String M = "M";
     public static final int MAX_NESTED_DEPTH = 1000;
 
     private static final String NUMBER = "<NUMBER>";
@@ -440,7 +442,8 @@ public static Geometry fromWKT(GeometryValidator validator, boolean coerce, Stri
      */
     private static Geometry parseGeometry(StreamTokenizer stream, boolean coerce, int depth) throws IOException, ParseException {
         final String type = nextWord(stream).toLowerCase(Locale.ROOT);
-        return switch (type) {
+        final boolean isExplicitlySpecifiesZorM = isZOrMNext(stream);
+        Geometry geometry = switch (type) {
             case "point" -> parsePoint(stream);
             case "multipoint" -> parseMultiPoint(stream);
             case "linestring" -> parseLine(stream);
@@ -453,6 +456,16 @@ private static Geometry parseGeometry(StreamTokenizer stream, boolean coerce, in
                 parseCircle(stream);
             default -> throw new IllegalArgumentException("Unknown geometry type: " + type);
         };
+        checkZorMAttribute(isExplicitlySpecifiesZorM, geometry.hasZ());
+        return geometry;
+    }
+
+    private static void checkZorMAttribute(boolean isExplicitlySpecifiesZorM, boolean hasZ) {
+        if (isExplicitlySpecifiesZorM && hasZ == false) {
+            throw new IllegalArgumentException(
+                "When specifying 'Z' or 'M', coordinates must include three values. Only two coordinates were provided"
+            );
+        }
     }
 
     private static GeometryCollection<Geometry> parseGeometryCollection(StreamTokenizer stream, boolean coerce, int depth)
@@ -710,6 +723,21 @@ private static boolean isNumberNext(StreamTokenizer stream) throws IOException {
         return type == StreamTokenizer.TT_WORD;
     }
 
+    private static boolean isZOrMNext(StreamTokenizer stream) {
+        String token;
+        try {
+            token = nextWord(stream);
+            if (token.equals(Z) || token.equals(M)) {
+                return true;
+            }
+            stream.pushBack();
+            return false;
+        } catch (ParseException | IOException e) {
+            return false;
+        }
+
+    }
+
     private static String nextEmptyOrOpen(StreamTokenizer stream) throws IOException, ParseException {
         final String next = nextWord(stream);
         if (next.equals(EMPTY) || next.equals(LPAREN)) {

docs/release-notes/breaking-changes.md

@@ -21,6 +21,9 @@
 
 abstract class BaseGeometryTestCase<T extends Geometry> extends AbstractWireTestCase<T> {
 
+    public static final String ZorMMustIncludeThreeValuesMsg =
+        "When specifying 'Z' or 'M', coordinates must include three values. Only two coordinates were provided";
+
     @Override
     protected final T createTestInstance() {
         boolean hasAlt = randomBoolean();

libs/entitlement/README.md

@@ -19,6 +19,7 @@
 import java.text.ParseException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.List;
 
 import static org.hamcrest.Matchers.containsString;
 
@@ -93,6 +94,35 @@ private int countNestedGeometryCollections(GeometryCollection<?> geometry) {
         return count;
     }
 
+    public void testParseGeometryCollectionZorMWithThreeCoordinates() throws IOException, ParseException {
+        GeometryValidator validator = GeographyValidator.instance(true);
+
+        GeometryCollection<Geometry> expected = new GeometryCollection<>(
+            Arrays.asList(
+                new Point(20.0, 10.0, 100.0),
+                new Line(new double[] { 10.0, 20.0 }, new double[] { 5.0, 15.0 }, new double[] { 50.0, 150.0 })
+            )
+        );
+
+        String point = "(POINT Z (20.0 10.0 100.0)";
+        String lineString = "LINESTRING M (10.0 5.0 50.0, 20.0 15.0 150.0)";
+        assertEquals(expected, WellKnownText.fromWKT(validator, true, "GEOMETRYCOLLECTION Z " + point + ", " + lineString + ")"));
+
+        assertEquals(expected, WellKnownText.fromWKT(validator, true, "GEOMETRYCOLLECTION M " + point + ", " + lineString + ")"));
+    }
+
+    public void testParseGeometryCollectionZorMWithTwoCoordinatesThrowsException() {
+        GeometryValidator validator = GeographyValidator.instance(true);
+        List<String> gcWkt = List.of(
+            "GEOMETRYCOLLECTION Z (POINT (20.0 10.0), LINESTRING (10.0 5.0, 20.0 15.0))",
+            "GEOMETRYCOLLECTION M (POINT (20.0 10.0), LINESTRING (10.0 5.0, 20.0 15.0))"
+        );
+        for (String gc : gcWkt) {
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> WellKnownText.fromWKT(validator, true, gc));
+            assertEquals(ZorMMustIncludeThreeValuesMsg, ex.getMessage());
+        }
+    }
+
     @Override
     protected GeometryCollection<Geometry> mutateInstance(GeometryCollection<Geometry> instance) {
         return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtils.java

@@ -17,6 +17,7 @@
 
 import java.io.IOException;
 import java.text.ParseException;
+import java.util.List;
 
 public class LineTests extends BaseGeometryTestCase<Line> {
     @Override
@@ -82,6 +83,25 @@ public void testWKTValidation() {
         assertEquals("found Z value [6.0] but [ignore_z_value] parameter is [false]", ex.getMessage());
     }
 
+    public void testParseLineZorMWithThreeCoordinates() throws IOException, ParseException {
+        GeometryValidator validator = GeographyValidator.instance(true);
+
+        Line expectedZ = new Line(new double[] { 20.0, 30.0 }, new double[] { 10.0, 15.0 }, new double[] { 100.0, 200.0 });
+        assertEquals(expectedZ, WellKnownText.fromWKT(validator, true, "LINESTRING Z (20.0 10.0 100.0, 30.0 15.0 200.0)"));
+
+        Line expectedM = new Line(new double[] { 20.0, 30.0 }, new double[] { 10.0, 15.0 }, new double[] { 100.0, 200.0 });
+        assertEquals(expectedM, WellKnownText.fromWKT(validator, true, "LINESTRING M (20.0 10.0 100.0, 30.0 15.0 200.0)"));
+    }
+
+    public void testParseLineZorMWithTwoCoordinatesThrowsException() {
+        GeometryValidator validator = GeographyValidator.instance(true);
+        List<String> linesWkt = List.of("LINESTRING Z (20.0 10.0, 30.0 15.0)", "LINESTRING M (20.0 10.0, 30.0 15.0)");
+        for (String line : linesWkt) {
+            IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> WellKnownText.fromWKT(validator, true, line));
+            assertEquals(ZorMMustIncludeThreeValuesMsg, ex.getMessage());
+        }
+    }
+
     @Override
     protected Line mutateInstance(Line instance) {
         return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/entitlements/FilesEntitlement.java

@@ -64,6 +64,54 @@ public void testValidation() {
             );
     }
 
+    public void testParseMultiLineZorMWithThreeCoordinates() throws IOException, ParseException {
+        GeometryValidator validator = GeographyValidator.instance(true);
+        MultiLine expectedZ = new MultiLine(
+            List.of(
+                new Line(new double[] { 20.0, 30.0 }, new double[] { 10.0, 15.0 }, new double[] { 100.0, 200.0 }),
+                new Line(new double[] { 40.0, 50.0 }, new double[] { 20.0, 25.0 }, new double[] { 300.0, 400.0 })
+            )
+        );
+        assertEquals(
+            expectedZ,
+            WellKnownText.fromWKT(
+                validator,
+                true,
+                "MULTILINESTRING Z ((20.0 10.0 100.0, 30.0 15.0 200.0), (40.0 20.0 300.0, 50.0 25.0 400.0))"
+            )
+        );
+
+        MultiLine expectedM = new MultiLine(
+            List.of(
+                new Line(new double[] { 20.0, 30.0 }, new double[] { 10.0, 15.0 }, new double[] { 100.0, 200.0 }),
+                new Line(new double[] { 40.0, 50.0 }, new double[] { 20.0, 25.0 }, new double[] { 300.0, 400.0 })
+            )
+        );
+        assertEquals(
+            expectedM,
+            WellKnownText.fromWKT(
+                validator,
+                true,
+                "MULTILINESTRING M ((20.0 10.0 100.0, 30.0 15.0 200.0), (40.0 20.0 300.0, 50.0 25.0 400.0))"
+            )
+        );
+    }
+
+    public void testParseMultiLineZorMWithTwoCoordinatesThrowsException() {
+        GeometryValidator validator = GeographyValidator.instance(true);
+        List<String> multiLinesWkt = List.of(
+            "MULTILINESTRING Z ((20.0 10.0, 30.0 15.0), (40.0 20.0, 50.0 25.0))",
+            "MULTILINESTRING M ((20.0 10.0, 30.0 15.0), (40.0 20.0, 50.0 25.0))"
+        );
+        for (String multiLine : multiLinesWkt) {
+            IllegalArgumentException ex = expectThrows(
+                IllegalArgumentException.class,
+                () -> WellKnownText.fromWKT(validator, true, multiLine)
+            );
+            assertEquals(ZorMMustIncludeThreeValuesMsg, ex.getMessage());
+        }
+    }
+
     @Override
     protected MultiLine mutateInstance(MultiLine instance) {
         return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929