Merge branch 'master' into feature/watcher-java-time-migration

Author: pgomulka

build.gradle

@@ -159,8 +159,8 @@ task verifyVersions {
  * the enabled state of every bwc task. It should be set back to true
  * after the backport of the backcompat code is complete.
  */
-final boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+final boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37899" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -150,7 +150,7 @@ class BuildPlugin implements Plugin<Project> {
             }
 
             String inFipsJvmScript = 'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));'
-            boolean inFipsJvm = Boolean.parseBoolean(runJavascript(project, runtimeJavaHome, inFipsJvmScript))
+            boolean inFipsJvm = Boolean.parseBoolean(runJavaAsScript(project, runtimeJavaHome, inFipsJvmScript))
 
             // Build debugging info
             println '======================================='
@@ -438,28 +438,28 @@ class BuildPlugin implements Plugin<Project> {
         String versionInfoScript = 'print(' +
             'java.lang.System.getProperty("java.vendor") + " " + java.lang.System.getProperty("java.version") + ' +
             '" [" + java.lang.System.getProperty("java.vm.name") + " " + java.lang.System.getProperty("java.vm.version") + "]");'
-        return runJavascript(project, javaHome, versionInfoScript).trim()
+        return runJavaAsScript(project, javaHome, versionInfoScript).trim()
     }
 
     /** Finds the parsable java specification version */
     private static String findJavaSpecificationVersion(Project project, String javaHome) {
         String versionScript = 'print(java.lang.System.getProperty("java.specification.version"));'
-        return runJavascript(project, javaHome, versionScript)
+        return runJavaAsScript(project, javaHome, versionScript)
     }
 
     private static String findJavaVendor(Project project, String javaHome) {
         String vendorScript = 'print(java.lang.System.getProperty("java.vendor"));'
-        return runJavascript(project, javaHome, vendorScript)
+        return runJavaAsScript(project, javaHome, vendorScript)
     }
 
     /** Finds the parsable java specification version */
     private static String findJavaVersion(Project project, String javaHome) {
         String versionScript = 'print(java.lang.System.getProperty("java.version"));'
-        return runJavascript(project, javaHome, versionScript)
+        return runJavaAsScript(project, javaHome, versionScript)
     }
 
     /** Runs the given javascript using jjs from the jdk, and returns the output */
-    private static String runJavascript(Project project, String javaHome, String script) {
+    private static String runJavaAsScript(Project project, String javaHome, String script) {
         ByteArrayOutputStream stdout = new ByteArrayOutputStream()
         ByteArrayOutputStream stderr = new ByteArrayOutputStream()
         if (Os.isFamily(Os.FAMILY_WINDOWS)) {

dev-tools/es_release_notes.pl

@@ -32,7 +32,7 @@
     ">enhancement", ">bug",           ">regression",  ">upgrade"
 );
 my %Ignore = map { $_ => 1 }
-    ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Build", "backport" );
+    ( ">non-issue", ">refactoring", ">docs", ">test", ">test-failure", ":Core/Infra/Build", "backport" );
 
 my %Group_Labels = (
     '>breaking'      => 'Breaking changes',
@@ -48,7 +48,7 @@
 
 my %Area_Overrides = (
     ':ml'            => 'Machine Learning',
-    ':beats'         => 'Beats Plugin',
+    ':Beats'         => 'Beats Plugin',
     ':Docs'          => 'Docs Infrastructure'
 );
 

docs/java-rest/high-level/getting-started.asciidoc

@@ -83,7 +83,7 @@ dependencies {
 The very first releases of any major version (like a beta), might have been built on top of a Lucene Snapshot version.
 In such a case you will be unable to resolve the Lucene dependencies of the client.
 
-For example, if you want to use the `7.0.0-alpha1` version which depends on Lucene `8.0.0-snapshot-6d9c714052`, you must
+For example, if you want to use the `7.0.0-alpha2` version which depends on Lucene `8.0.0-snapshot-774e9aefbc`, you must
 define the following repository.
 
 For Maven:
@@ -93,7 +93,7 @@ For Maven:
 <repository>
     <id>elastic-lucene-snapshots</id>
     <name>Elastic Lucene Snapshots</name>
-    <url>http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/6d9c714052</url>
+    <url>http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/774e9aefbc</url>
     <releases><enabled>true</enabled></releases>
     <snapshots><enabled>false</enabled></snapshots>
 </repository>
@@ -104,7 +104,7 @@ For Gradle:
 ["source","groovy",subs="attributes"]
 --------------------------------------------------
 maven {
-    url 'http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/6d9c714052'
+    url 'http://s3.amazonaws.com/download.elasticsearch.org/lucenesnapshots/774e9aefbc'
 }
 --------------------------------------------------
 

docs/reference/mapping/removal_of_types.asciidoc

@@ -606,5 +606,5 @@ PUT index-2-01
 
 In case of implicit index creation, because of documents that get indexed in
 an index that doesn't exist yet, the template is always honored. This is
-usually not a problem due to the fact that typless index calls work on typed
+usually not a problem due to the fact that typeless index calls work on typed
 indices.

docs/reference/sql/endpoints/jdbc.asciidoc

@@ -115,8 +115,6 @@ Query timeout (in seconds). That is the maximum amount of time waiting for a que
 
 `ssl.truststore.pass`:: trust store password
 
-`ssl.cert.allow.self.signed` (default `false`):: Whether or not to allow self signed certificates
-
 `ssl.protocol`(default `TLS`):: SSL protocol to be used
 
 [float]

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -708,7 +708,8 @@ protected static void configureClient(RestClientBuilder builder, Settings settin
                 throw new IllegalStateException(TRUSTSTORE_PATH + " is set but points to a non-existing file");
             }
             try {
-                KeyStore keyStore = KeyStore.getInstance("jks");
+                final String keyStoreType = keystorePath.endsWith(".p12") ? "PKCS12" : "jks";
+                KeyStore keyStore = KeyStore.getInstance(keyStoreType);
                 try (InputStream is = Files.newInputStream(path)) {
                     keyStore.load(is, keystorePass.toCharArray());
                 }

x-pack/plugin/sql/jdbc/src/test/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfigurationTests.java

@@ -6,9 +6,16 @@
 package org.elasticsearch.xpack.sql.jdbc;
 
 import org.elasticsearch.test.ESTestCase;
+import org.elasticsearch.xpack.sql.client.SslConfig;
 
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Properties;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.CONNECT_TIMEOUT;
 import static org.elasticsearch.xpack.sql.client.ConnectionConfiguration.PAGE_TIMEOUT;
@@ -130,5 +137,153 @@ public void testTimoutOverride() throws Exception {
         assertThat(ci.pageTimeout(), equalTo(4L));
     }
 
-
+    public void testSSLPropertiesInUrl() throws Exception {
+        Map<String, String> urlPropMap = sslProperties();
+        
+        Properties allProps = new Properties();
+        allProps.putAll(urlPropMap);
+        String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&"));
+        
+        assertSslConfig(allProps, ci("jdbc:es://test?" + sslUrlProps.toString()).sslConfig());
+    }
+    
+    public void testSSLPropertiesInUrlAndProperties() throws Exception {
+        Map<String, String> urlPropMap = new HashMap<>(4);
+        urlPropMap.put("ssl", "false");
+        urlPropMap.put("ssl.protocol", "SSLv3");
+        urlPropMap.put("ssl.keystore.location", "/abc/xyz");
+        urlPropMap.put("ssl.keystore.pass", "mypass");
+        
+        Map<String, String> propMap = new HashMap<>(4);
+        propMap.put("ssl.keystore.type", "PKCS12");
+        propMap.put("ssl.truststore.location", "/foo/bar");
+        propMap.put("ssl.truststore.pass", "anotherpass");
+        propMap.put("ssl.truststore.type", "jks");
+        
+        Properties props = new Properties();
+        props.putAll(propMap);
+        String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&"));
+        
+        Properties allProps = new Properties();
+        allProps.putAll(urlPropMap);
+        allProps.putAll(propMap);
+        assertSslConfig(allProps, JdbcConfiguration.create("jdbc:es://test?" + sslUrlProps.toString(), props, 0).sslConfig());
+    }
+    
+    public void testSSLPropertiesOverride() throws Exception {
+        Map<String, String> urlPropMap = sslProperties();
+        Map<String, String> propMap = new HashMap<>(8);
+        propMap.put("ssl", "false");
+        propMap.put("ssl.protocol", "TLS");
+        propMap.put("ssl.keystore.location", "/xyz");
+        propMap.put("ssl.keystore.pass", "different_mypass");
+        propMap.put("ssl.keystore.type", "JKS");
+        propMap.put("ssl.truststore.location", "/baz");
+        propMap.put("ssl.truststore.pass", "different_anotherpass");
+        propMap.put("ssl.truststore.type", "PKCS11");
+        
+        Properties props = new Properties();
+        props.putAll(propMap);
+        String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&"));
+        assertSslConfig(props, JdbcConfiguration.create("jdbc:es://test?" + sslUrlProps.toString(), props, 0).sslConfig());
+    }
+    
+    public void testDriverConfigurationWithSSLInURL() {
+        Map<String, String> urlPropMap = sslProperties();
+        
+        Properties allProps = new Properties();
+        allProps.putAll(urlPropMap);
+        String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&"));
+        
+        try {
+            DriverManager.getDriver("jdbc:es://test?" + sslUrlProps);
+        } catch (SQLException sqle) {
+            fail("Driver registration should have been successful. Error: " + sqle);
+        }
+    }
+    
+    public void testDataSourceConfigurationWithSSLInURL() throws SQLException, URISyntaxException {
+        Map<String, String> urlPropMap = sslProperties();
+        
+        Properties allProps = new Properties();
+        allProps.putAll(urlPropMap);
+        String sslUrlProps = urlPropMap.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining("&"));
+        
+        EsDataSource dataSource = new EsDataSource();
+        String address = "jdbc:es://test?" + sslUrlProps;
+        dataSource.setUrl(address);
+        JdbcConnection connection = null;
+        
+        try {
+            connection = (JdbcConnection) dataSource.getConnection();
+        } catch (SQLException sqle) {
+            fail("Connection creation should have been successful. Error: " + sqle);
+        }
+        
+        assertEquals(address, connection.getURL());
+        assertSslConfig(allProps, connection.cfg.sslConfig());
+    }
+    
+    public void testTyposInSslConfigInUrl(){
+        assertJdbcSqlExceptionFromUrl("ssl.protocl", "ssl.protocol");
+        assertJdbcSqlExceptionFromUrl("sssl", "ssl");
+        assertJdbcSqlExceptionFromUrl("ssl.keystore.lction", "ssl.keystore.location");
+        assertJdbcSqlExceptionFromUrl("ssl.keystore.pss", "ssl.keystore.pass");
+        assertJdbcSqlExceptionFromUrl("ssl.keystore.typ", "ssl.keystore.type");
+        assertJdbcSqlExceptionFromUrl("ssl.trustsore.location", "ssl.truststore.location");
+        assertJdbcSqlExceptionFromUrl("ssl.tuststore.pass", "ssl.truststore.pass");
+        assertJdbcSqlExceptionFromUrl("ssl.ruststore.type", "ssl.truststore.type");
+    }
+    
+    public void testTyposInSslConfigInProperties() {
+        assertJdbcSqlExceptionFromProperties("ssl.protocl", "ssl.protocol");
+        assertJdbcSqlExceptionFromProperties("sssl", "ssl");
+        assertJdbcSqlExceptionFromProperties("ssl.keystore.lction", "ssl.keystore.location");
+        assertJdbcSqlExceptionFromProperties("ssl.keystore.pss", "ssl.keystore.pass");
+        assertJdbcSqlExceptionFromProperties("ssl.keystore.typ", "ssl.keystore.type");
+        assertJdbcSqlExceptionFromProperties("ssl.trustsore.location", "ssl.truststore.location");
+        assertJdbcSqlExceptionFromProperties("ssl.tuststore.pass", "ssl.truststore.pass");
+        assertJdbcSqlExceptionFromProperties("ssl.ruststore.type", "ssl.truststore.type");
+    }
+    
+    private Map<String, String> sslProperties() {
+        Map<String, String> sslPropertiesMap = new HashMap<>(8);
+        // always using "false" so that the SSLContext doesn't actually start verifying the keystore and trustore
+        // locations, as we don't have file permissions to access them.
+        sslPropertiesMap.put("ssl", "false");
+        sslPropertiesMap.put("ssl.protocol", "SSLv3");
+        sslPropertiesMap.put("ssl.keystore.location", "/abc/xyz");
+        sslPropertiesMap.put("ssl.keystore.pass", "mypass");
+        sslPropertiesMap.put("ssl.keystore.type", "PKCS12");
+        sslPropertiesMap.put("ssl.truststore.location", "/foo/bar");
+        sslPropertiesMap.put("ssl.truststore.pass", "anotherpass");
+        sslPropertiesMap.put("ssl.truststore.type", "jks");
+        
+        return sslPropertiesMap;
+    }
+    
+    private void assertSslConfig(Properties allProperties, SslConfig sslConfig) throws URISyntaxException {
+        // because SslConfig doesn't expose its internal properties (and it shouldn't),
+        // we compare a newly created SslConfig with the one from the JdbcConfiguration with the equals() method
+        SslConfig mockSslConfig = new SslConfig(allProperties, new URI("http://test:9200/"));
+        assertEquals(mockSslConfig, sslConfig);
+    }
+    
+    private void assertJdbcSqlExceptionFromUrl(String wrongSetting, String correctSetting) {
+        String url = "jdbc:es://test?" + wrongSetting + "=foo";
+        assertJdbcSqlException(wrongSetting, correctSetting, url, null);
+    }
+    
+    private void assertJdbcSqlExceptionFromProperties(String wrongSetting, String correctSetting) {
+        String url = "jdbc:es://test";
+        Properties props = new Properties();
+        props.put(wrongSetting, correctSetting);
+        assertJdbcSqlException(wrongSetting, correctSetting, url, props);
+    }
+    
+    private void assertJdbcSqlException(String wrongSetting, String correctSetting, String url, Properties props) {
+        JdbcSQLException ex = expectThrows(JdbcSQLException.class, 
+                () -> JdbcConfiguration.create(url, props, 0));
+        assertEquals("Unknown parameter [" + wrongSetting + "] ; did you mean [" + correctSetting + "]", ex.getMessage());
+    }
 }

x-pack/plugin/sql/qa/security/with-ssl/build.gradle

@@ -208,7 +208,7 @@ integTestCluster {
   }
 }
 Closure notRunningFips = {
-  Boolean.parseBoolean(BuildPlugin.runJavascript(project, project.runtimeJavaHome,
+  Boolean.parseBoolean(BuildPlugin.runJavaAsScript(project, project.runtimeJavaHome,
           'print(java.security.Security.getProviders()[0].name.toLowerCase().contains("fips"));')) == false
 }
 

x-pack/plugin/sql/sql-client/src/main/java/org/elasticsearch/xpack/sql/client/SslConfig.java

@@ -63,7 +63,7 @@ public class SslConfig {
 
     private final SSLContext sslContext;
 
-    SslConfig(Properties settings, URI baseURI) {
+    public SslConfig(Properties settings, URI baseURI) {
         boolean isSchemaPresent = baseURI.getScheme() != null;
         boolean isSSLPropertyPresent = settings.getProperty(SSL) != null;
         boolean isHttpsScheme = "https".equals(baseURI.getScheme());