Skip to content

Commit c0b2a70

Browse files
committed
Merge remote-tracking branch 'origin/master' into snapshot-lifecycle-management
2 parents 4321b24 + 1f019eb commit c0b2a70

File tree

188 files changed

+5110
-1796
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

188 files changed

+5110
-1796
lines changed

TESTING.asciidoc

+5-6
Original file line numberDiff line numberDiff line change
@@ -340,13 +340,13 @@ be downloaded again unless they have been updated to a new version.
340340
+
341341
. Run the tests with `./gradlew packagingTest`. This will cause Gradle to build
342342
the tar, zip, and deb packages and all the plugins. It will then run the tests
343-
on ubuntu-1404 and centos-7. We chose those two distributions as the default
343+
on ubuntu-1604 and centos-7. We chose those two distributions as the default
344344
because they cover deb and rpm packaging and SyvVinit and systemd.
345345

346346
You can choose which boxes to test by setting the `-Pvagrant.boxes` project property. All of
347347
the valid options for this property are:
348348

349-
* `sample` - The default, only chooses ubuntu-1404 and centos-7
349+
* `sample` - The default, only chooses ubuntu-1604 and centos-7
350350
* List of box names, comma separated (e.g. `oel-7,fedora-28`) - Chooses exactly the boxes listed.
351351
* `linux-all` - All linux boxes.
352352
* `windows-all` - All Windows boxes. If there are any Windows boxes which do not
@@ -364,11 +364,10 @@ will remain running and you'll have to stop them manually with `./gradlew stop`
364364

365365
All the regular vagrant commands should just work so you can get a shell in a
366366
VM running trusty by running
367-
`vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404`.
367+
`vagrant up ubuntu-1604 --provider virtualbox && vagrant ssh ubuntu-1604`.
368368

369369
These are the linux flavors supported, all of which we provide images for
370370

371-
* ubuntu-1404 aka trusty
372371
* ubuntu-1604 aka xenial
373372
* ubuntu-1804 aka bionic beaver
374373
* debian-8 aka jessie
@@ -422,13 +421,13 @@ It's important to think of VMs like cattle. If they become lame you just shoot
422421
them and let vagrant reprovision them. Say you've hosed your precise VM:
423422

424423
----------------------------------------------------
425-
vagrant ssh ubuntu-1404 -c 'sudo rm -rf /bin'; echo oops
424+
vagrant ssh ubuntu-1604 -c 'sudo rm -rf /bin'; echo oops
426425
----------------------------------------------------
427426

428427
All you've got to do to get another one is
429428

430429
----------------------------------------------
431-
vagrant destroy -f ubuntu-1404 && vagrant up ubuntu-1404 --provider virtualbox
430+
vagrant destroy -f ubuntu-1604 && vagrant up ubuntu-1604 --provider virtualbox
432431
----------------------------------------------
433432

434433
The whole process takes a minute and a half on a modern laptop, two and a half

Vagrantfile

-6
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,6 @@ Vagrant.configure(2) do |config|
4646
PROJECT_DIR = ENV['VAGRANT_PROJECT_DIR'] || Dir.pwd
4747
config.vm.synced_folder PROJECT_DIR, '/project'
4848

49-
'ubuntu-1404'.tap do |box|
50-
config.vm.define box, define_opts do |config|
51-
config.vm.box = 'elastic/ubuntu-14.04-x86_64'
52-
deb_common config, box
53-
end
54-
end
5549
'ubuntu-1604'.tap do |box|
5650
config.vm.define box, define_opts do |config|
5751
config.vm.box = 'elastic/ubuntu-16.04-x86_64'

build.gradle

+2-2
Original file line numberDiff line numberDiff line change
@@ -162,8 +162,8 @@ task verifyVersions {
162162
* after the backport of the backcompat code is complete.
163163
*/
164164

165-
boolean bwc_tests_enabled = false
166-
final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/40319" /* place a PR link here when committing bwc changes */
165+
boolean bwc_tests_enabled = true
166+
final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
167167
if (bwc_tests_enabled == false) {
168168
if (bwc_tests_disabled_issue.isEmpty()) {
169169
throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/build.gradle

+1
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,7 @@ if (project != rootProject) {
245245

246246
forbiddenPatterns {
247247
exclude '**/*.wav'
248+
exclude '**/*.p12'
248249
// the file that actually defines nocommit
249250
exclude '**/ForbiddenPatternsTask.java'
250251
}

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

+2-10
Original file line numberDiff line numberDiff line change
@@ -361,16 +361,8 @@ class BuildPlugin implements Plugin<Project> {
361361
compilerJavaHome = findJavaHome(compilerJavaProperty)
362362
}
363363
if (compilerJavaHome == null) {
364-
if (System.getProperty("idea.active") != null || System.getProperty("eclipse.launcher") != null) {
365-
// IntelliJ does not set JAVA_HOME, so we use the JDK that Gradle was run with
366-
return Jvm.current().javaHome
367-
} else {
368-
throw new GradleException(
369-
"JAVA_HOME must be set to build Elasticsearch. " +
370-
"Note that if the variable was just set you might have to run `./gradlew --stop` for " +
371-
"it to be picked up. See https://github.com/elastic/elasticsearch/issues/31399 details."
372-
)
373-
}
364+
// if JAVA_HOME does not set,so we use the JDK that Gradle was run with.
365+
return Jvm.current().javaHome
374366
}
375367
return compilerJavaHome
376368
}

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

+1-2
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ class VagrantTestPlugin implements Plugin<Project> {
3131
'oel-7',
3232
'opensuse-42',
3333
'sles-12',
34-
'ubuntu-1404',
3534
'ubuntu-1604',
3635
'ubuntu-1804'
3736
])
@@ -48,7 +47,7 @@ class VagrantTestPlugin implements Plugin<Project> {
4847
/** Boxes used when sampling the tests **/
4948
static final List<String> SAMPLE = unmodifiableList([
5049
'centos-7',
51-
'ubuntu-1404'
50+
'ubuntu-1604'
5251
])
5352

5453
/** All distributions to bring into test VM, whether or not they are used **/
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.gradle.http;
21+
22+
import org.gradle.api.logging.Logger;
23+
import org.gradle.api.logging.Logging;
24+
25+
import javax.net.ssl.HttpsURLConnection;
26+
import javax.net.ssl.KeyManager;
27+
import javax.net.ssl.SSLContext;
28+
import javax.net.ssl.TrustManagerFactory;
29+
import java.io.File;
30+
import java.io.FileInputStream;
31+
import java.io.IOException;
32+
import java.io.InputStream;
33+
import java.net.HttpURLConnection;
34+
import java.net.MalformedURLException;
35+
import java.net.URL;
36+
import java.nio.charset.StandardCharsets;
37+
import java.security.GeneralSecurityException;
38+
import java.security.KeyStore;
39+
import java.security.KeyStoreException;
40+
import java.security.SecureRandom;
41+
import java.security.cert.Certificate;
42+
import java.security.cert.CertificateFactory;
43+
import java.util.Arrays;
44+
import java.util.Base64;
45+
import java.util.Collections;
46+
import java.util.Enumeration;
47+
import java.util.HashSet;
48+
import java.util.Set;
49+
import java.util.concurrent.TimeUnit;
50+
51+
/**
52+
* A utility to wait for a specific HTTP resource to be available, optionally with customized TLS trusted CAs.
53+
* This is logically similar to using the Ant Get task to retrieve a resource, but with the difference that it can
54+
* access resources that do not use the JRE's default trusted CAs.
55+
*/
56+
public class WaitForHttpResource {
57+
58+
private static final Logger logger = Logging.getLogger(WaitForHttpResource.class);
59+
60+
private Set<Integer> validResponseCodes = Collections.singleton(200);
61+
private URL url;
62+
private Set<File> certificateAuthorities;
63+
private File trustStoreFile;
64+
private String trustStorePassword;
65+
private String username;
66+
private String password;
67+
68+
public WaitForHttpResource(String protocol, String host, int numberOfNodes) throws MalformedURLException {
69+
this(new URL(protocol + "://" + host + "/_cluster/health?wait_for_nodes=>=" + numberOfNodes + "&wait_for_status=yellow"));
70+
}
71+
72+
public WaitForHttpResource(URL url) {
73+
this.url = url;
74+
}
75+
76+
public void setValidResponseCodes(int... validResponseCodes) {
77+
this.validResponseCodes = new HashSet<>(validResponseCodes.length);
78+
for (int rc : validResponseCodes) {
79+
this.validResponseCodes.add(rc);
80+
}
81+
}
82+
83+
public void setCertificateAuthorities(File... certificateAuthorities) {
84+
this.certificateAuthorities = new HashSet<>(Arrays.asList(certificateAuthorities));
85+
}
86+
87+
public void setTrustStoreFile(File trustStoreFile) {
88+
this.trustStoreFile = trustStoreFile;
89+
}
90+
91+
public void setTrustStorePassword(String trustStorePassword) {
92+
this.trustStorePassword = trustStorePassword;
93+
}
94+
95+
public void setUsername(String username) {
96+
this.username = username;
97+
}
98+
99+
public void setPassword(String password) {
100+
this.password = password;
101+
}
102+
103+
public boolean wait(int durationInMs) throws GeneralSecurityException, InterruptedException, IOException {
104+
final long waitUntil = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationInMs);
105+
final long sleep = Long.max(durationInMs / 10, 100);
106+
107+
final SSLContext ssl;
108+
final KeyStore trustStore = buildTrustStore();
109+
if (trustStore != null) {
110+
ssl = createSslContext(trustStore);
111+
} else {
112+
ssl = null;
113+
}
114+
IOException failure = null;
115+
for (; ; ) {
116+
try {
117+
checkResource(ssl);
118+
return true;
119+
} catch (IOException e) {
120+
logger.debug("Failed to access resource [{}]", url, e);
121+
failure = e;
122+
}
123+
if (System.nanoTime() < waitUntil) {
124+
Thread.sleep(sleep);
125+
} else {
126+
logger.error("Failed to access url [{}]", url, failure);
127+
return false;
128+
}
129+
}
130+
}
131+
132+
protected void checkResource(SSLContext ssl) throws IOException {
133+
try {
134+
final HttpURLConnection connection = buildConnection(ssl);
135+
connection.connect();
136+
final Integer response = connection.getResponseCode();
137+
if (validResponseCodes.contains(response)) {
138+
logger.info("Got successful response [{}] from URL [{}]", response, url);
139+
return;
140+
} else {
141+
throw new IOException(response + " " + connection.getResponseMessage());
142+
}
143+
} catch (IOException e) {
144+
throw e;
145+
}
146+
}
147+
148+
HttpURLConnection buildConnection(SSLContext ssl) throws IOException {
149+
final HttpURLConnection connection = (HttpURLConnection) this.url.openConnection();
150+
configureSslContext(connection, ssl);
151+
configureBasicAuth(connection);
152+
connection.setRequestMethod("GET");
153+
return connection;
154+
}
155+
156+
private void configureSslContext(HttpURLConnection connection, SSLContext ssl) {
157+
if (ssl != null) {
158+
if (connection instanceof HttpsURLConnection) {
159+
((HttpsURLConnection) connection).setSSLSocketFactory(ssl.getSocketFactory());
160+
} else {
161+
throw new IllegalStateException("SSL trust has been configured, but [" + url + "] is not a 'https' URL");
162+
}
163+
}
164+
}
165+
166+
private void configureBasicAuth(HttpURLConnection connection) {
167+
if (username != null) {
168+
if (password == null) {
169+
throw new IllegalStateException("Basic Auth user [" + username
170+
+ "] has been set, but no password has been configured");
171+
}
172+
connection.setRequestProperty("Authorization",
173+
"Basic " + Base64.getEncoder().encodeToString((username + ":" + password).getBytes(StandardCharsets.UTF_8)));
174+
}
175+
}
176+
177+
KeyStore buildTrustStore() throws GeneralSecurityException, IOException {
178+
if (this.certificateAuthorities != null) {
179+
if (trustStoreFile != null) {
180+
throw new IllegalStateException("Cannot specify both truststore and CAs");
181+
}
182+
return buildTrustStoreFromCA();
183+
} else if (trustStoreFile != null) {
184+
return buildTrustStoreFromFile();
185+
} else {
186+
return null;
187+
}
188+
}
189+
190+
private KeyStore buildTrustStoreFromFile() throws GeneralSecurityException, IOException {
191+
KeyStore keyStore = KeyStore.getInstance(trustStoreFile.getName().endsWith(".jks") ? "JKS" : "PKCS12");
192+
try (InputStream input = new FileInputStream(trustStoreFile)) {
193+
keyStore.load(input, trustStorePassword == null ? null : trustStorePassword.toCharArray());
194+
}
195+
return keyStore;
196+
}
197+
198+
private KeyStore buildTrustStoreFromCA() throws GeneralSecurityException, IOException {
199+
final KeyStore store = KeyStore.getInstance(KeyStore.getDefaultType());
200+
store.load(null, null);
201+
final CertificateFactory certFactory = CertificateFactory.getInstance("X.509");
202+
int counter = 0;
203+
for (File ca : certificateAuthorities) {
204+
try (InputStream input = new FileInputStream(ca)) {
205+
for (Certificate certificate : certFactory.generateCertificates(input)) {
206+
store.setCertificateEntry("cert-" + counter, certificate);
207+
counter++;
208+
}
209+
}
210+
}
211+
return store;
212+
}
213+
214+
private SSLContext createSslContext(KeyStore trustStore) throws GeneralSecurityException {
215+
checkForTrustEntry(trustStore);
216+
TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
217+
tmf.init(trustStore);
218+
SSLContext sslContext = SSLContext.getInstance("TLSv1.2");
219+
sslContext.init(new KeyManager[0], tmf.getTrustManagers(), new SecureRandom());
220+
return sslContext;
221+
}
222+
223+
private void checkForTrustEntry(KeyStore trustStore) throws KeyStoreException {
224+
Enumeration<String> enumeration = trustStore.aliases();
225+
while (enumeration.hasMoreElements()) {
226+
if (trustStore.isCertificateEntry(enumeration.nextElement())) {
227+
// found trusted cert entry
228+
return;
229+
}
230+
}
231+
throw new IllegalStateException("Trust-store does not contain any trusted certificate entries");
232+
}
233+
}

0 commit comments

Comments
 (0)