Skip to content

Commit 9c177bd

Browse files
henningandersenhenningandersen
committed
Merge remote-tracking branch 'origin/master' into fix_reindex_disallow_partial_search_results
2 parents 3a5853e + a0967a4 commit 9c177bd

File tree

1,178 files changed

+30392
-12708
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,178 files changed

+30392
-12708
lines changed

‎.ci/bwcVersions

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,7 @@ BWC_VERSION:
77
- "7.2.1"
88
- "7.3.0"
99
- "7.3.1"
10+
- "7.3.2"
1011
- "7.4.0"
12+
- "7.5.0"
1113
- "8.0.0"

‎.ci/java-versions.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# are 'java' or 'openjdk' followed by the major release number.
66

77
ES_BUILD_JAVA=openjdk12
8-
ES_RUNTIME_JAVA=java11
8+
ES_RUNTIME_JAVA=openjdk11
99
GRADLE_TASK=build
1010

1111

‎.ci/matrix-runtime-javas.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ ES_RUNTIME_JAVA:
1010
- java12
1111
- openjdk12
1212
- openjdk13
13+
- openjdk14
1314
- zulu11
1415
- zulu12
1516
- corretto11
17+
- adoptopenjdk11

‎build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
3333

3434
plugins {
3535
id 'com.gradle.build-scan' version '2.4'
36-
id 'base'
36+
id 'lifecycle-base'
3737
id 'elasticsearch.global-build-info'
3838
}
3939

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier
4848
import org.gradle.api.artifacts.ProjectDependency
4949
import org.gradle.api.artifacts.ResolvedArtifact
5050
import org.gradle.api.artifacts.dsl.RepositoryHandler
51-
import org.gradle.api.artifacts.repositories.ArtifactRepository
5251
import org.gradle.api.artifacts.repositories.IvyArtifactRepository
5352
import org.gradle.api.artifacts.repositories.IvyPatternRepositoryLayout
5453
import org.gradle.api.artifacts.repositories.MavenArtifactRepository
@@ -85,12 +84,10 @@ import org.gradle.util.GradleVersion
8584

8685
import java.nio.charset.StandardCharsets
8786
import java.nio.file.Files
88-
import java.time.ZoneOffset
89-
import java.time.ZonedDateTime
9087
import java.util.regex.Matcher
9188

92-
import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
9389
import static org.elasticsearch.gradle.tool.Boilerplate.findByName
90+
import static org.elasticsearch.gradle.tool.Boilerplate.maybeConfigure
9491

9592
/**
9693
* Encapsulates build configuration for elasticsearch projects.
@@ -411,11 +408,11 @@ class BuildPlugin implements Plugin<Project> {
411408
project.getRepositories().all { repository ->
412409
if (repository instanceof MavenArtifactRepository) {
413410
final MavenArtifactRepository maven = (MavenArtifactRepository) repository
414-
assertRepositoryURIUsesHttps(maven, project, maven.getUrl())
415-
repository.getArtifactUrls().each { uri -> assertRepositoryURIUsesHttps(maven, project, uri) }
411+
assertRepositoryURIIsSecure(maven.name, project.path, maven.getUrl())
412+
repository.getArtifactUrls().each { uri -> assertRepositoryURIIsSecure(maven.name, project.path, uri) }
416413
} else if (repository instanceof IvyArtifactRepository) {
417414
final IvyArtifactRepository ivy = (IvyArtifactRepository) repository
418-
assertRepositoryURIUsesHttps(ivy, project, ivy.getUrl())
415+
assertRepositoryURIIsSecure(ivy.name, project.path, ivy.getUrl())
419416
}
420417
}
421418
RepositoryHandler repos = project.repositories
@@ -455,9 +452,15 @@ class BuildPlugin implements Plugin<Project> {
455452
}
456453
}
457454

458-
private static void assertRepositoryURIUsesHttps(final ArtifactRepository repository, final Project project, final URI uri) {
459-
if (uri != null && uri.toURL().getProtocol().equals("http")) {
460-
throw new GradleException("repository [${repository.name}] on project with path [${project.path}] is using http for artifacts on [${uri.toURL()}]")
455+
static void assertRepositoryURIIsSecure(final String repositoryName, final String projectPath, final URI uri) {
456+
if (uri != null && ["file", "https", "s3"].contains(uri.getScheme()) == false) {
457+
final String message = String.format(
458+
Locale.ROOT,
459+
"repository [%s] on project with path [%s] is not using a secure protocol for artifacts on [%s]",
460+
repositoryName,
461+
projectPath,
462+
uri.toURL())
463+
throw new GradleException(message)
461464
}
462465
}
463466

@@ -884,6 +887,12 @@ class BuildPlugin implements Plugin<Project> {
884887
// TODO: remove this once cname is prepended to transport.publish_address by default in 8.0
885888
test.systemProperty 'es.transport.cname_in_publish_address', 'true'
886889

890+
// Set netty system properties to the properties we configure in jvm.options
891+
test.systemProperty('io.netty.noUnsafe', 'true')
892+
test.systemProperty('io.netty.noKeySetOptimization', 'true')
893+
test.systemProperty('io.netty.recycler.maxCapacityPerThread', '0')
894+
test.systemProperty('io.netty.allocator.numDirectArenas', '0')
895+
887896
test.testLogging { TestLoggingContainer logging ->
888897
logging.showExceptions = true
889898
logging.showCauses = true

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/RestTestsFromSnippetsTask.groovy

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -202,11 +202,12 @@ public class RestTestsFromSnippetsTask extends SnippetsTask {
202202
previousTest = snippet
203203
return
204204
}
205-
if (snippet.testResponse) {
205+
if (snippet.testResponse || snippet.language == 'console-result') {
206206
response(snippet)
207207
return
208208
}
209-
if (snippet.test || snippet.console) {
209+
if (snippet.test || snippet.console ||
210+
snippet.language == 'console') {
210211
test(snippet)
211212
previousTest = snippet
212213
return

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/SnippetsTask.groovy

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ public class SnippetsTask extends DefaultTask {
123123
}
124124
}
125125
if (snippet.testResponse
126-
&& 'js' == snippet.language
126+
&& ('js' == snippet.language || 'console-result' == snippet.language)
127127
&& null == snippet.skip) {
128128
String quoted = snippet.contents
129129
// quote values starting with $
@@ -162,7 +162,7 @@ public class SnippetsTask extends DefaultTask {
162162
}
163163
return
164164
}
165-
matcher = line =~ /\["?source"?,\s*"?(\w+)"?(,.*)?].*/
165+
matcher = line =~ /\["?source"?,\s*"?([-\w]+)"?(,.*)?].*/
166166
if (matcher.matches()) {
167167
lastLanguage = matcher.group(1)
168168
lastLanguageLine = lineNumber

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -370,7 +370,7 @@ class ClusterFormationTasks {
370370
Map esConfig = [
371371
'cluster.name' : node.clusterName,
372372
'node.name' : "node-" + node.nodeNum,
373-
'pidfile' : node.pidFile,
373+
(node.nodeVersion.onOrAfter('7.4.0') ? 'node.pidfile' : 'pidfile') : node.pidFile,
374374
'path.repo' : "${node.sharedDir}/repo",
375375
'path.shared_data' : "${node.sharedDir}/",
376376
// Define a node attribute so we can test that it exists

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,7 @@ class RestIntegTestTask extends DefaultTask {
236236
}
237237
project.dependencies {
238238
restSpec ClasspathUtils.isElasticsearchProject() ? project.project(':rest-api-spec') :
239-
"org.elasticsearch.rest-api-spec:${VersionProperties.elasticsearch}"
239+
"org.elasticsearch:rest-api-spec:${VersionProperties.elasticsearch}"
240240
}
241241
Task copyRestSpec = project.tasks.findByName('copyRestSpec')
242242
if (copyRestSpec != null) {

‎buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -464,13 +464,6 @@ private void logToProcessStdout(String message) {
464464
public void restart() {
465465
LOGGER.info("Restarting {}", this);
466466
stop(false);
467-
try {
468-
Files.delete(httpPortsFile);
469-
Files.delete(transportPortFile);
470-
} catch (IOException e) {
471-
throw new UncheckedIOException(e);
472-
}
473-
474467
start();
475468
}
476469

@@ -479,11 +472,11 @@ public void goToNextVersion() {
479472
if (currentDistro + 1 >= distributions.size()) {
480473
throw new TestClustersException("Ran out of versions to go to for " + this);
481474
}
482-
LOGGER.info("Switch version from {} to {} for {}",
483-
getVersion(), distributions.get(currentDistro + 1).getVersion(), this
484-
);
475+
logToProcessStdout("Switch version from " + getVersion() + " to " + distributions.get(currentDistro + 1).getVersion());
476+
stop(false);
485477
currentDistro += 1;
486-
restart();
478+
setting("node.attr.upgraded", "true");
479+
start();
487480
}
488481

489482
private boolean isSettingMissingOrTrue(String name) {
@@ -716,6 +709,17 @@ public File getAuditLog() {
716709

717710
@Override
718711
public synchronized void stop(boolean tailLogs) {
712+
logToProcessStdout("Stopping node");
713+
try {
714+
if (Files.exists(httpPortsFile)) {
715+
Files.delete(httpPortsFile);
716+
}
717+
if (Files.exists(transportPortFile)) {
718+
Files.delete(transportPortFile);
719+
}
720+
} catch (IOException e) {
721+
throw new UncheckedIOException(e);
722+
}
719723
if (esProcess == null && tailLogs) {
720724
// This is a special case. If start() throws an exception the plugin will still call stop
721725
// Another exception here would eat the orriginal.
@@ -867,6 +871,8 @@ private void waitForProcessToExit(ProcessHandle processHandle) {
867871

868872
private void createWorkingDir(Path distroExtractDir) throws IOException {
869873
syncWithLinks(distroExtractDir, distroDir);
874+
// Start configuration from scratch in case of a restart
875+
project.delete(configFile.getParent());
870876
Files.createDirectories(configFile.getParent());
871877
Files.createDirectories(confPathRepo);
872878
Files.createDirectories(confPathData);
@@ -958,6 +964,17 @@ private void createConfiguration() {
958964
// Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
959965
defaultConfig.put("discovery.initial_state_timeout", "0s");
960966

967+
// TODO: Remove these once https://github.com/elastic/elasticsearch/issues/46091 is fixed
968+
defaultConfig.put("logger.org.elasticsearch.action.support.master.TransportMasterNodeAction", "TRACE");
969+
defaultConfig.put("logger.org.elasticsearch.cluster.metadata.MetaDataCreateIndexService", "TRACE");
970+
defaultConfig.put("logger.org.elasticsearch.cluster.service", "DEBUG");
971+
defaultConfig.put("logger.org.elasticsearch.cluster.coordination", "DEBUG");
972+
defaultConfig.put("logger.org.elasticsearch.gateway.MetaStateService", "TRACE");
973+
if (getVersion().getMajor() >= 8) {
974+
defaultConfig.put("cluster.service.slow_task_logging_threshold", "5s");
975+
defaultConfig.put("cluster.service.slow_master_task_logging_threshold", "5s");
976+
}
977+
961978
HashSet<String> overriden = new HashSet<>(defaultConfig.keySet());
962979
overriden.retainAll(settings.keySet());
963980
overriden.removeAll(OVERRIDABLE_SETTINGS);

‎buildSrc/src/minimumRuntime/java/org/elasticsearch/gradle/VersionProperties.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@ public static String getElasticsearch() {
1515
return elasticsearch;
1616
}
1717

18+
public static Version getElasticsearchVersion() {
19+
return Version.fromString(elasticsearch);
20+
}
21+
1822
public static String getLucene() {
1923
return lucene;
2024
}

‎buildSrc/src/test/groovy/org/elasticsearch/gradle/BuildPluginTests.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222
import org.gradle.api.GradleException;
2323
import org.junit.Test;
2424

25+
import java.net.URI;
26+
import java.net.URISyntaxException;
27+
2528

2629
public class BuildPluginTests extends GradleUnitTestCase {
2730

@@ -36,4 +39,25 @@ public void testFailingDockerVersions() {
3639
BuildPlugin.checkDockerVersionRecent("Docker version 17.04.0, build e68fc7a");
3740
}
3841

42+
@Test(expected = GradleException.class)
43+
public void testRepositoryURIThatUsesHttpScheme() throws URISyntaxException {
44+
final URI uri = new URI("http://s3.amazonaws.com/artifacts.elastic.co/maven");
45+
BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
46+
}
47+
48+
public void testRepositoryThatUsesFileScheme() throws URISyntaxException {
49+
final URI uri = new URI("file:/tmp/maven");
50+
BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
51+
}
52+
53+
public void testRepositoryURIThatUsesHttpsScheme() throws URISyntaxException {
54+
final URI uri = new URI("https://s3.amazonaws.com/artifacts.elastic.co/maven");
55+
BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
56+
}
57+
58+
public void testRepositoryURIThatUsesS3Scheme() throws URISyntaxException {
59+
final URI uri = new URI("s3://artifacts.elastic.co/maven");
60+
BuildPlugin.assertRepositoryURIIsSecure("test", "test", uri);
61+
}
62+
3963
}

‎buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginIT.java

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,29 @@
1818
*/
1919
package org.elasticsearch.gradle;
2020

21+
import org.apache.commons.io.FileUtils;
2122
import org.apache.commons.io.IOUtils;
2223
import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
2324
import org.gradle.testkit.runner.BuildResult;
25+
import org.gradle.testkit.runner.GradleRunner;
26+
import org.junit.Rule;
27+
import org.junit.rules.TemporaryFolder;
2428

2529
import java.io.File;
2630
import java.io.IOException;
2731
import java.io.InputStream;
2832
import java.nio.charset.StandardCharsets;
33+
import java.nio.file.Files;
34+
import java.util.Arrays;
35+
import java.util.List;
2936
import java.util.zip.ZipEntry;
3037
import java.util.zip.ZipFile;
3138

3239
public class BuildPluginIT extends GradleIntegrationTestCase {
3340

41+
@Rule
42+
public TemporaryFolder tmpDir = new TemporaryFolder();
43+
3444
public void testPluginCanBeApplied() {
3545
BuildResult result = getGradleRunner("elasticsearch.build")
3646
.withArguments("hello", "-s")
@@ -46,6 +56,51 @@ public void testCheckTask() {
4656
assertTaskSuccessful(result, ":check");
4757
}
4858

59+
public void testInsecureMavenRepository() throws IOException {
60+
final String name = "elastic-maven";
61+
final String url = "http://s3.amazonaws.com/artifacts.elastic.co/maven";
62+
// add an insecure maven repository to the build.gradle
63+
final List<String> lines = Arrays.asList(
64+
"repositories {",
65+
" maven {",
66+
" name \"elastic-maven\"",
67+
" url \"" + url + "\"\n",
68+
" }",
69+
"}");
70+
runInsecureArtifactRepositoryTest(name, url, lines);
71+
}
72+
73+
public void testInsecureIvyRepository() throws IOException {
74+
final String name = "elastic-ivy";
75+
final String url = "http://s3.amazonaws.com/artifacts.elastic.co/ivy";
76+
// add an insecure ivy repository to the build.gradle
77+
final List<String> lines = Arrays.asList(
78+
"repositories {",
79+
" ivy {",
80+
" name \"elastic-ivy\"",
81+
" url \"" + url + "\"\n",
82+
" }",
83+
"}");
84+
runInsecureArtifactRepositoryTest(name, url, lines);
85+
}
86+
87+
private void runInsecureArtifactRepositoryTest(final String name, final String url, final List<String> lines) throws IOException {
88+
final File projectDir = getProjectDir("elasticsearch.build");
89+
FileUtils.copyDirectory(projectDir, tmpDir.getRoot(), pathname -> pathname.getPath().contains("/build/") == false);
90+
final List<String> buildGradleLines =
91+
Files.readAllLines(tmpDir.getRoot().toPath().resolve("build.gradle"), StandardCharsets.UTF_8);
92+
buildGradleLines.addAll(lines);
93+
Files.write(tmpDir.getRoot().toPath().resolve("build.gradle"), buildGradleLines, StandardCharsets.UTF_8);
94+
final BuildResult result = GradleRunner.create()
95+
.withProjectDir(tmpDir.getRoot())
96+
.withArguments("clean", "hello", "-s", "-i", "--warning-mode=all", "--scan")
97+
.withPluginClasspath()
98+
.buildAndFail();
99+
assertOutputContains(
100+
result.getOutput(),
101+
"repository [" + name + "] on project with path [:] is not using a secure protocol for artifacts on [" + url + "]");
102+
}
103+
49104
public void testLicenseAndNotice() throws IOException {
50105
BuildResult result = getGradleRunner("elasticsearch.build")
51106
.withArguments("clean", "assemble", "-s", "-Dlocal.repo.path=" + getLocalTestRepoPath())

‎buildSrc/version.properties

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ slf4j = 1.6.2
2121
jna = 4.5.1
2222

2323
netty = 4.1.38.Final
24-
joda = 2.10.2
24+
joda = 2.10.3
2525

2626
# when updating this version, you need to ensure compatibility with:
2727
# - plugins/ingest-attachment (transitive dependency, check the upstream POM)

0 commit comments

Comments
 (0)