Skip to content

Commit 35c3de1

Browse files
github-actions[bot]github-actions[bot]
authored
Merge pull request #162 from elastic/master
🤖 ESQL: Merge upstream
2 parents dc55969 + 1cef841 commit 35c3de1

File tree

89 files changed

+4088
-465
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+4088
-465
lines changed

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradlePrecommitPluginFuncTest.groovy renamed to ‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleInternalPluginFuncTest.groovy

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,11 @@
88

99
package org.elasticsearch.gradle.fixtures
1010

11-
import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin
11+
import org.gradle.api.Plugin
1212

13-
abstract class AbstractGradlePrecommitPluginFuncTest extends AbstractJavaGradleFuncTest {
13+
abstract class AbstractGradleInternalPluginFuncTest extends AbstractJavaGradleFuncTest {
1414

15-
abstract <T extends PrecommitPlugin> Class<T> getPluginClassUnderTest();
15+
abstract <T extends Plugin> Class<T> getPluginClassUnderTest();
1616

1717
def setup() {
1818
buildFile << """

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/LicenseHeadersPrecommitPluginFuncTest.groovy

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,22 @@
88

99
package org.elasticsearch.gradle.internal.precommit
1010

11-
import org.elasticsearch.gradle.fixtures.AbstractGradlePrecommitPluginFuncTest
11+
import org.elasticsearch.gradle.fixtures.AbstractGradleInternalPluginFuncTest
1212
import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersPrecommitPlugin
1313
import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin
1414
import org.gradle.testkit.runner.TaskOutcome
1515

16-
class LicenseHeadersPrecommitPluginFuncTest extends AbstractGradlePrecommitPluginFuncTest {
16+
class LicenseHeadersPrecommitPluginFuncTest extends AbstractGradleInternalPluginFuncTest {
1717

1818
Class<? extends PrecommitPlugin> pluginClassUnderTest = LicenseHeadersPrecommitPlugin.class
1919

2020
def setup() {
21+
configurationCacheCompatible = true
2122
buildFile << """
2223
apply plugin:'java'
2324
"""
2425
}
26+
2527
def "detects invalid files with invalid license header"() {
2628
given:
2729
dualLicensedFile()

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/precommit/TestingConventionsPrecommitPluginFuncTest.groovy

Lines changed: 3 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
package org.elasticsearch.gradle.internal.precommit
1010

11-
import org.elasticsearch.gradle.fixtures.AbstractGradlePrecommitPluginFuncTest
11+
import org.elasticsearch.gradle.fixtures.AbstractGradleInternalPluginFuncTest
1212
import org.elasticsearch.gradle.fixtures.LocalRepositoryFixture
1313
import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin
1414
import org.gradle.testkit.runner.TaskOutcome
@@ -20,7 +20,7 @@ import spock.lang.Unroll
2020

2121
// see https://github.com/elastic/elasticsearch/issues/87913
2222
@IgnoreIf({ os.windows })
23-
class TestingConventionsPrecommitPluginFuncTest extends AbstractGradlePrecommitPluginFuncTest {
23+
class TestingConventionsPrecommitPluginFuncTest extends AbstractGradleInternalPluginFuncTest {
2424

2525
Class<? extends PrecommitPlugin> pluginClassUnderTest = TestingConventionsPrecommitPlugin.class
2626

@@ -45,6 +45,7 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradlePrecommitP
4545
}
4646

4747
def setup() {
48+
configurationCacheCompatible = true
4849
repository.configureBuild(buildFile)
4950
}
5051

@@ -84,26 +85,6 @@ class TestingConventionsPrecommitPluginFuncTest extends AbstractGradlePrecommitP
8485
result.task(":testingConventions").outcome == TaskOutcome.UP_TO_DATE
8586
}
8687

87-
def "testing convention plugin is configuration cache compatible"() {
88-
given:
89-
simpleJavaBuild()
90-
testClazz("org.acme.valid.SomeTests", "org.apache.lucene.tests.util.LuceneTestCase") {
91-
"""
92-
public void testMe() {
93-
}
94-
"""
95-
}
96-
when:
97-
def result = gradleRunner("precommit", "--configuration-cache").build()
98-
then:
99-
assertOutputContains(result.getOutput(), "0 problems were found storing the configuration cache.")
100-
101-
when:
102-
result = gradleRunner("precommit", "--configuration-cache").build()
103-
then:
104-
assertOutputContains(result.getOutput(), "Configuration cache entry reused.")
105-
}
106-
10788
def "checks base class convention"() {
10889
given:
10990
simpleJavaBuild()

‎build-tools/src/testFixtures/groovy/org/elasticsearch/gradle/fixtures/AbstractGradleFuncTest.groovy

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
package org.elasticsearch.gradle.fixtures
1010

1111
import org.apache.commons.io.FileUtils
12+
import org.elasticsearch.gradle.internal.test.ConfigurationCacheCompatibleAwareGradleRunner
1213
import org.elasticsearch.gradle.internal.test.InternalAwareGradleRunner
1314
import org.elasticsearch.gradle.internal.test.NormalizeOutputGradleRunner
1415
import org.gradle.testkit.runner.BuildResult
@@ -33,6 +34,8 @@ abstract class AbstractGradleFuncTest extends Specification {
3334
File propertiesFile
3435
File projectDir
3536

37+
boolean configurationCacheCompatible = false
38+
3639
def setup() {
3740
projectDir = testProjectDir.root
3841
settingsFile = testProjectDir.newFile('settings.gradle')
@@ -71,14 +74,17 @@ abstract class AbstractGradleFuncTest extends Specification {
7174

7275
GradleRunner gradleRunner(File projectDir, String... arguments) {
7376
return new NormalizeOutputGradleRunner(
74-
new InternalAwareGradleRunner(
75-
GradleRunner.create()
76-
.withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments().toString().indexOf("-agentlib:jdwp") > 0)
77-
.withProjectDir(projectDir)
78-
.withPluginClasspath()
79-
.forwardOutput()
80-
),
81-
projectDir
77+
new ConfigurationCacheCompatibleAwareGradleRunner(
78+
new InternalAwareGradleRunner(
79+
GradleRunner.create()
80+
.withDebug(ManagementFactory.getRuntimeMXBean().getInputArguments()
81+
.toString().indexOf("-agentlib:jdwp") > 0
82+
)
83+
.withProjectDir(projectDir)
84+
.withPluginClasspath()
85+
.forwardOutput()
86+
), configurationCacheCompatible),
87+
projectDir
8288
).withArguments(arguments)
8389
}
8490

@@ -127,13 +133,13 @@ abstract class AbstractGradleFuncTest extends Specification {
127133
}
128134

129135
File internalBuild(
130-
List<String> extraPlugins = [],
131-
String bugfix = "7.15.2",
132-
String bugfixLucene = "8.9.0",
133-
String staged = "7.16.0",
134-
String stagedLucene = "8.10.0",
135-
String minor = "8.0.0",
136-
String minorLucene = "9.0.0"
136+
List<String> extraPlugins = [],
137+
String bugfix = "7.15.2",
138+
String bugfixLucene = "8.9.0",
139+
String staged = "7.16.0",
140+
String stagedLucene = "8.10.0",
141+
String minor = "8.0.0",
142+
String minorLucene = "9.0.0"
137143
) {
138144
buildFile << """plugins {
139145
id 'elasticsearch.global-build-info'

‎build-tools/src/testFixtures/java/org/elasticsearch/gradle/internal/test/ConfigurationCacheCompatibleAwareGradleRunner.java

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
9+
package org.elasticsearch.gradle.internal.test;
10+
11+
import org.gradle.testkit.runner.BuildResult;
12+
import org.gradle.testkit.runner.GradleRunner;
13+
import org.gradle.testkit.runner.InvalidPluginMetadataException;
14+
import org.gradle.testkit.runner.InvalidRunnerConfigurationException;
15+
import org.gradle.testkit.runner.UnexpectedBuildFailure;
16+
import org.gradle.testkit.runner.UnexpectedBuildSuccess;
17+
18+
import java.io.File;
19+
import java.io.Writer;
20+
import java.net.URI;
21+
import java.util.ArrayList;
22+
import java.util.List;
23+
import java.util.Map;
24+
25+
/**
26+
* A Gradle runner that delegates to another runner, optionally enabling the configuring cache parameter.
27+
*/
28+
public class ConfigurationCacheCompatibleAwareGradleRunner extends GradleRunner {
29+
private GradleRunner delegate;
30+
private boolean ccCompatible;
31+
32+
public ConfigurationCacheCompatibleAwareGradleRunner(GradleRunner delegate, boolean ccCompatible) {
33+
this.delegate = delegate;
34+
this.ccCompatible = ccCompatible;
35+
}
36+
37+
@Override
38+
public GradleRunner withGradleVersion(String gradleVersion) {
39+
delegate.withGradleVersion(gradleVersion);
40+
return this;
41+
}
42+
43+
@Override
44+
public GradleRunner withGradleInstallation(File file) {
45+
delegate.withGradleInstallation(file);
46+
return this;
47+
}
48+
49+
@Override
50+
public GradleRunner withGradleDistribution(URI uri) {
51+
delegate.withGradleDistribution(uri);
52+
return this;
53+
}
54+
55+
@Override
56+
public GradleRunner withTestKitDir(File file) {
57+
delegate.withTestKitDir(file);
58+
return this;
59+
}
60+
61+
@Override
62+
public File getProjectDir() {
63+
return delegate.getProjectDir();
64+
}
65+
66+
@Override
67+
public GradleRunner withProjectDir(File projectDir) {
68+
delegate.withProjectDir(projectDir);
69+
return this;
70+
}
71+
72+
@Override
73+
public List<String> getArguments() {
74+
return delegate.getArguments();
75+
}
76+
77+
@Override
78+
public GradleRunner withArguments(List<String> arguments) {
79+
List<String> effectiveArgs = arguments;
80+
if (ccCompatible) {
81+
effectiveArgs = new ArrayList<>(arguments);
82+
effectiveArgs.add("--configuration-cache");
83+
}
84+
delegate.withArguments(effectiveArgs);
85+
return this;
86+
}
87+
88+
@Override
89+
public GradleRunner withArguments(String... arguments) {
90+
withArguments(List.of(arguments));
91+
return this;
92+
}
93+
94+
@Override
95+
public List<? extends File> getPluginClasspath() {
96+
return delegate.getPluginClasspath();
97+
}
98+
99+
@Override
100+
public GradleRunner withPluginClasspath() throws InvalidPluginMetadataException {
101+
delegate.withPluginClasspath();
102+
return this;
103+
}
104+
105+
@Override
106+
public GradleRunner withPluginClasspath(Iterable<? extends File> iterable) {
107+
delegate.withPluginClasspath(iterable);
108+
return this;
109+
}
110+
111+
@Override
112+
public boolean isDebug() {
113+
return delegate.isDebug();
114+
}
115+
116+
@Override
117+
public GradleRunner withDebug(boolean b) {
118+
delegate.withDebug(b);
119+
return this;
120+
}
121+
122+
@Override
123+
public Map<String, String> getEnvironment() {
124+
return delegate.getEnvironment();
125+
}
126+
127+
@Override
128+
public GradleRunner withEnvironment(Map<String, String> map) {
129+
delegate.withEnvironment(map);
130+
return this;
131+
}
132+
133+
@Override
134+
public GradleRunner forwardStdOutput(Writer writer) {
135+
delegate.forwardStdOutput(writer);
136+
return this;
137+
}
138+
139+
@Override
140+
public GradleRunner forwardStdError(Writer writer) {
141+
delegate.forwardStdOutput(writer);
142+
return this;
143+
}
144+
145+
@Override
146+
public GradleRunner forwardOutput() {
147+
delegate.forwardOutput();
148+
return this;
149+
}
150+
151+
@Override
152+
public BuildResult build() throws InvalidRunnerConfigurationException, UnexpectedBuildFailure {
153+
return delegate.build();
154+
}
155+
156+
@Override
157+
public BuildResult buildAndFail() throws InvalidRunnerConfigurationException, UnexpectedBuildSuccess {
158+
return delegate.buildAndFail();
159+
}
160+
}

‎docs/changelog/87719.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 87719
2+
summary: Don't ignore pipeline for upserts in bulk api
3+
area: Ingest
4+
type: bug
5+
issues:
6+
- 87131

‎docs/changelog/87793.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 87793
2+
summary: Geoip processor should respect the `ignore_missing` in case of missing database
3+
area: Ingest
4+
type: bug
5+
issues:
6+
- 87345

‎docs/changelog/88007.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 88007
2+
summary: Warn about impact of large readahead on search
3+
area: Performance
4+
type: enhancement
5+
issues: []

‎docs/reference/api-conventions.asciidoc

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ If provided, {es} surfaces the header's `trace-id` value as `trace.id` in the:
5656
* <<deprecation-logging,Deprecation logs>>
5757

5858
For example, the following `traceparent` value would produce the following
59-
`trade.id` value in the above logs.
59+
`trace.id` value in the above logs.
6060

6161
[source,txt]
6262
----

‎docs/reference/how-to/search-speed.asciidoc

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,32 @@ fast. In general, you should make sure that at least half the available memory
99
goes to the filesystem cache so that Elasticsearch can keep hot regions of the
1010
index in physical memory.
1111

12+
[discrete]
13+
=== Avoid page cache thrashing by using modest readahead values on Linux
14+
15+
Search can cause a lot of randomized read I/O. When the underlying block
16+
device has a high readahead value, there may be a lot of unnecessary
17+
read I/O done, especially when files are accessed using memory mapping
18+
(see <<file-system,storage types>>).
19+
20+
Most Linux distributions use a sensible readahead value of `128KiB` for a
21+
single plain device, however, when using software raid, LVM or dm-crypt the
22+
resulting block device (backing Elasticsearch <<path-settings,path.data>>)
23+
may end up having a very large readahead value (in the range of several MiB).
24+
This usually results in severe page (filesystem) cache thrashing adversely
25+
affecting search (or <<docs,update>>) performance.
26+
27+
You can check the current value in `KiB` using
28+
`lsblk -o NAME,RA,MOUNTPOINT,TYPE,SIZE`.
29+
Consult the documentation of your distribution on how to alter this value
30+
(for example with a `udev` rule to persist across reboots, or via
31+
https://man7.org/linux/man-pages/man8/blockdev.8.html[blockdev --setra]
32+
as a transient setting). We recommend a value of `128KiB` for readahead.
33+
34+
WARNING: `blockdev` expects values in 512 byte sectors whereas `lsblk` reports
35+
values in `KiB`. As an example, to temporarily set readahead to `128KiB`
36+
for `/dev/nvme0n1`, specify `blockdev --setra 256 /dev/nvme0n1`.
37+
1238
[discrete]
1339
=== Use faster hardware
1440

0 commit comments

Comments
 (0)