Skip to content

Commit 4ad6b51

Browse files
williamrandolphwilliamrandolph
committed
Merge branch 'master' into feature-pwd-protected-keystore
2 parents 0f34e9f + b78053c commit 4ad6b51

File tree

459 files changed

+13398
-4867
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

459 files changed

+13398
-4867
lines changed

‎.ci/build-cache.gradle

Lines changed: 0 additions & 18 deletions
This file was deleted.

‎.ci/bwcVersions

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@ BWC_VERSION:
66
- "7.2.0"
77
- "7.2.1"
88
- "7.3.0"
9+
- "7.3.1"
910
- "7.4.0"
1011
- "8.0.0"

‎.ci/init.gradle

Lines changed: 88 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,97 @@
1-
if (System.env.ELASTIC_ARTIFACTORY_USERNAME == null || System.env.ELASTIC_ARTIFACTORY_TOKEN == null) {
2-
throw new GradleException("Using init script without configuration")
3-
} else {
4-
logger.info("Using elastic artifactory repos")
5-
settingsEvaluated { settings ->
6-
settings.pluginManagement {
1+
import com.bettercloud.vault.VaultConfig;
2+
import com.bettercloud.vault.Vault;
3+
4+
initscript {
5+
repositories {
6+
mavenCentral()
7+
}
8+
dependencies {
9+
classpath 'com.bettercloud:vault-java-driver:4.1.0'
10+
}
11+
}
12+
13+
['VAULT_ADDR', 'VAULT_ROLE_ID', 'VAULT_SECRET_ID'].each {
14+
if (System.env."$it" == null) {
15+
throw new GradleException("$it must be set!")
16+
17+
}
18+
}
19+
20+
final String vaultToken = new Vault(
21+
new VaultConfig()
22+
.address(System.env.VAULT_ADDR)
23+
.engineVersion(1)
24+
.build()
25+
)
26+
.withRetries(5, 1000)
27+
.auth()
28+
.loginByAppRole("approle", System.env.VAULT_ROLE_ID, System.env.VAULT_SECRET_ID)
29+
.getAuthClientToken();
30+
31+
final Vault vault = new Vault(
32+
new VaultConfig()
33+
.address(System.env.VAULT_ADDR)
34+
.engineVersion(1)
35+
.token(vaultToken)
36+
.build()
37+
)
38+
.withRetries(5, 1000)
39+
40+
final Map<String,String> artifactoryCredentials = vault.logical()
41+
.read("secret/elasticsearch-ci/artifactory.elstc.co")
42+
.getData();
43+
44+
logger.info("Using elastic artifactory repos")
45+
Closure configCache = {
46+
return {
47+
name "artifactory-gradle-release"
48+
url "https://artifactory.elstc.co/artifactory/gradle-release"
49+
credentials {
50+
username artifactoryCredentials.get("username")
51+
password artifactoryCredentials.get("token")
52+
}
53+
}
54+
}
55+
settingsEvaluated { settings ->
56+
settings.pluginManagement {
57+
repositories {
58+
maven configCache()
59+
}
60+
}
61+
}
62+
projectsLoaded {
63+
allprojects {
64+
buildscript {
765
repositories {
8-
maven {
9-
name "artifactory-gradle-plugins"
10-
url "https://artifactory.elstc.co/artifactory/gradle-plugins"
11-
credentials {
12-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
13-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
14-
}
15-
}
16-
gradlePluginPortal()
66+
maven configCache()
1767
}
1868
}
69+
repositories {
70+
maven configCache()
71+
}
1972
}
20-
projectsLoaded {
21-
allprojects {
22-
buildscript {
23-
repositories {
24-
maven {
25-
name "artifactory-gradle-release"
26-
url "https://artifactory.elstc.co/artifactory/gradle-release/"
27-
credentials {
28-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
29-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
30-
}
31-
}
32-
}
33-
}
34-
repositories {
35-
maven {
36-
name "artifactory-gradle-release"
37-
url "https://artifactory.elstc.co/artifactory/gradle-release/"
38-
credentials {
39-
username System.env.ELASTIC_ARTIFACTORY_USERNAME
40-
password System.env.ELASTIC_ARTIFACTORY_TOKEN
41-
}
73+
rootProject {
74+
project.pluginManager.withPlugin('com.gradle.build-scan') {
75+
buildScan.server = 'https://gradle-enterprise.elastic.co'
76+
}
77+
}
78+
}
79+
80+
if (System.env.GRADLE_BUILD_CACHE_URL != null) {
81+
final Map<String,String> buildCacheCredentials = vault.logical()
82+
.read("secret/elasticsearch-ci/gradle-build-cache")
83+
.getData();
84+
gradle.settingsEvaluated { settings ->
85+
settings.buildCache {
86+
remote(HttpBuildCache) {
87+
url = System.getenv('GRADLE_BUILD_CACHE_URL')
88+
push = Boolean.valueOf(System.getenv('GRADLE_BUILD_CACHE_PUSH') ?: 'false')
89+
credentials {
90+
username = buildCacheCredentials.get("username")
91+
password = buildCacheCredentials.get("password")
4292
}
4393
}
4494
}
4595
}
4696
}
97+

‎README.textile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,9 +155,9 @@ Complete control on the index level is allowed. As an example, in the above case
155155
<pre>
156156
curl -XPUT http://localhost:9200/another_user?pretty -H 'Content-Type: application/json' -d '
157157
{
158-
"index" : {
159-
"number_of_shards" : 2,
160-
"number_of_replicas" : 1
158+
"settings" : {
159+
"index.number_of_shards" : 2,
160+
"index.number_of_replicas" : 1
161161
}
162162
}'
163163
</pre>

‎buildSrc/build.gradle

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,22 +66,30 @@ if (JavaVersion.current() < JavaVersion.VERSION_11) {
6666
throw new GradleException('At least Java 11 is required to build elasticsearch gradle tools')
6767
}
6868

69-
// Keep compatibility with Java 8 for external users of build-tools that haven't migrated to Java 11
70-
targetCompatibility = '11'
71-
sourceCompatibility = '11'
69+
allprojects {
70+
targetCompatibility = '11'
71+
sourceCompatibility = '11'
72+
}
7273

7374
sourceSets {
7475
// We have a few classes that need to be compiled for older java versions
7576
minimumRuntime { }
7677
}
7778

79+
configurations {
80+
reaper
81+
}
82+
7883
compileMinimumRuntimeJava {
7984
targetCompatibility = 8
8085
sourceCompatibility = 8
8186
}
8287

8388
jar {
8489
from sourceSets.minimumRuntime.output
90+
into('META-INF') {
91+
from configurations.reaper
92+
}
8593
}
8694

8795
javadoc {
@@ -119,6 +127,7 @@ dependencies {
119127
testCompile "junit:junit:${props.getProperty('junit')}"
120128
testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
121129
testCompile 'com.github.tomakehurst:wiremock-jre8-standalone:2.23.2'
130+
reaper project('reaper')
122131
minimumRuntimeCompile "junit:junit:${props.getProperty('junit')}"
123132
minimumRuntimeCompile localGroovy()
124133
minimumRuntimeCompile gradleApi()

‎buildSrc/reaper/build.gradle

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
apply plugin: 'java'
2+
3+
jar {
4+
manifest {
5+
attributes 'Main-Class': 'org.elasticsearch.gradle.reaper.Reaper'
6+
}
7+
}

‎buildSrc/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.gradle.reaper;
21+
22+
import java.io.Closeable;
23+
import java.io.IOException;
24+
import java.io.UncheckedIOException;
25+
import java.nio.file.Files;
26+
import java.nio.file.Path;
27+
import java.nio.file.Paths;
28+
import java.util.Comparator;
29+
import java.util.List;
30+
import java.util.stream.Collectors;
31+
import java.util.stream.Stream;
32+
33+
/**
34+
* A standalone process that will reap external services after a build dies.
35+
*
36+
* <h2>Input</h2>
37+
* Since how to reap a given service is platform and service dependent, this tool
38+
* operates on system commands to execute. It takes a single argument, a directory
39+
* that will contain files with reaping commands. Each line in each file will be
40+
* executed with {@link Runtime#getRuntime()#exec}.
41+
*
42+
* The main method will wait indefinitely on the parent process (Gradle) by
43+
* reading from stdin. When Gradle shuts down, whether normally or abruptly, the
44+
* pipe will be broken and read will return.
45+
*
46+
* The reaper will then iterate over the files in the configured directory,
47+
* and execute the given commands. If any commands fail, a failure message is
48+
* written to stderr. Otherwise, the input file will be deleted. If no inputs
49+
* produced errors, the entire input directory is deleted upon completion of reaping.
50+
*/
51+
public class Reaper implements Closeable {
52+
53+
private Path inputDir;
54+
private boolean failed;
55+
56+
private Reaper(Path inputDir) {
57+
this.inputDir = inputDir;
58+
this.failed = false;
59+
}
60+
61+
public static void main(String[] args) throws Exception {
62+
if (args.length != 1) {
63+
System.err.println("Expected one argument.\nUsage: java -jar reaper.jar <DIR_OF_REAPING_COMMANDS>");
64+
System.exit(1);
65+
}
66+
Path inputDir = Paths.get(args[0]);
67+
68+
try (Reaper reaper = new Reaper(inputDir)){
69+
System.in.read();
70+
reaper.reap();
71+
}
72+
}
73+
74+
private void reap() {
75+
try (Stream<Path> stream = Files.list(inputDir)){
76+
final List<Path> inputFiles = stream.filter(p -> p.getFileName().toString().endsWith(".cmd")).collect(Collectors.toList());
77+
78+
for (Path inputFile : inputFiles) {
79+
System.out.println("Process file: " + inputFile);
80+
String line = Files.readString(inputFile);
81+
System.out.println("Running command: " + line);
82+
String[] command = line.split(" ");
83+
Process process = Runtime.getRuntime().exec(command);
84+
int ret = process.waitFor();
85+
86+
System.out.print("Stdout: ");
87+
process.getInputStream().transferTo(System.out);
88+
System.out.print("\nStderr: ");
89+
process.getErrorStream().transferTo(System.out);
90+
System.out.println(); // end the stream
91+
if (ret != 0) {
92+
logFailure("Command [" + line + "] failed with exit code " + ret, null);
93+
} else {
94+
delete(inputFile);
95+
}
96+
}
97+
} catch (Exception e) {
98+
logFailure("Failed to reap inputs", e);
99+
}
100+
}
101+
102+
private void logFailure(String message, Exception e) {
103+
System.err.println(message);
104+
if (e != null) {
105+
e.printStackTrace(System.err);
106+
}
107+
failed = true;
108+
}
109+
110+
private void delete(Path toDelete) {
111+
try {
112+
Files.delete(toDelete);
113+
} catch (IOException e) {
114+
logFailure("Failed to delete [" + toDelete + "]", e);
115+
}
116+
}
117+
118+
@Override
119+
public void close() {
120+
if (failed == false) {
121+
try (Stream<Path> stream = Files.walk(inputDir)){
122+
stream.sorted(Comparator.reverseOrder()).forEach(this::delete);
123+
} catch (IOException e) {
124+
throw new UncheckedIOException(e);
125+
}
126+
}
127+
}
128+
}

‎buildSrc/settings.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
include 'reaper'

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
2424
import groovy.transform.CompileDynamic
2525
import groovy.transform.CompileStatic
2626
import org.apache.commons.io.IOUtils
27-
import org.eclipse.jgit.lib.Constants
28-
import org.eclipse.jgit.lib.RepositoryBuilder
2927
import org.elasticsearch.gradle.info.GlobalBuildInfoPlugin
3028
import org.elasticsearch.gradle.info.GlobalInfoExtension
3129
import org.elasticsearch.gradle.info.JavaHome
@@ -711,26 +709,12 @@ class BuildPlugin implements Plugin<Project> {
711709
// after the doFirst added by the info plugin, and we can override attributes
712710
JavaVersion compilerJavaVersion = ext.get('compilerJavaVersion') as JavaVersion
713711
jarTask.manifest.attributes(
712+
'Change': ext.get('gitRevision'),
714713
'X-Compile-Elasticsearch-Version': VersionProperties.elasticsearch,
715714
'X-Compile-Lucene-Version': VersionProperties.lucene,
716715
'X-Compile-Elasticsearch-Snapshot': VersionProperties.isElasticsearchSnapshot(),
717-
'Build-Date': ZonedDateTime.now(ZoneOffset.UTC),
716+
'Build-Date': ext.get('buildDate'),
718717
'Build-Java-Version': compilerJavaVersion)
719-
if (jarTask.manifest.attributes.containsKey('Change') == false) {
720-
jarTask.logger.warn('Building without git revision id.')
721-
jarTask.manifest.attributes('Change': 'Unknown')
722-
} else {
723-
/*
724-
* The info-scm plugin assumes that if GIT_COMMIT is set it was set by Jenkins to the commit hash for this build.
725-
* However, that assumption is wrong as this build could be a sub-build of another Jenkins build for which GIT_COMMIT
726-
* is the commit hash for that build. Therefore, if GIT_COMMIT is set we calculate the commit hash ourselves.
727-
*/
728-
if (System.getenv("GIT_COMMIT") != null) {
729-
final String hash = new RepositoryBuilder().findGitDir(project.buildDir).build().resolve(Constants.HEAD).name
730-
final String shortHash = hash?.substring(0, 7)
731-
jarTask.manifest.attributes('Change': shortHash)
732-
}
733-
}
734718
}
735719

736720
// add license/notice files

0 commit comments

Comments
 (0)