Run active directory tests against a samba4 fixture (elastic/x-pack-elasticsearch#4067)

This commit adds a Samba4 test fixture that acts as a domain controller
and has the same contents as the cloud active directory instance that
we previously used for tests.

The tests also support reading information from environment variables
so that they can be run against a real active directory instance in our
CI builds.

In addition, this commit also fixes a few issues that surfaced when
making this change. The first is a change in the base DN that is
searched when performing down-level authentication. The base DN is
now the configuration object instead of the domain DN. This change was
required due to the original producing unnecessary referrals, which we
cannot easily follow when running against this test figure. Referrals
cannot easily be followed as they are returned by the ldap server with
an unresolvable DNS name unless the host points to the samba4 instance
for DNS. The port returned in the referral url is the one samba is bound
to, which differs from the port that is forwarded to the host by the
test fixture.

The other issue that is resolved by this change is the addition of
settings that allow specifying non-standard ports for active directory.
This is needed for down-level authentication as we may need to query
the regular port of active directory instead of the global catalog
port as the configuration object is not replicated to the global
catalog.

relates elastic/x-pack-elasticsearch#185
Relates elastic/x-pack-elasticsearch#3800

Original commit: elastic/x-pack-elasticsearch@883c742

Author: jaymode

plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/ldap/ActiveDirectorySessionFactorySettings.java

@@ -20,6 +20,10 @@ public final class ActiveDirectorySessionFactorySettings {
     public static final String AD_UPN_USER_SEARCH_FILTER_SETTING = "user_search.upn_filter";
     public static final String AD_DOWN_LEVEL_USER_SEARCH_FILTER_SETTING = "user_search.down_level_filter";
     public static final String AD_USER_SEARCH_SCOPE_SETTING = "user_search.scope";
+    public static final Setting<Integer> AD_LDAP_PORT_SETTING = Setting.intSetting("port.ldap", 389, Setting.Property.NodeScope);
+    public static final Setting<Integer> AD_LDAPS_PORT_SETTING = Setting.intSetting("port.ldaps", 636, Setting.Property.NodeScope);
+    public static final Setting<Integer> AD_GC_LDAP_PORT_SETTING = Setting.intSetting("port.gc_ldap", 3268, Setting.Property.NodeScope);
+    public static final Setting<Integer> AD_GC_LDAPS_PORT_SETTING = Setting.intSetting("port.gc_ldaps", 3269, Setting.Property.NodeScope);
     public static final Setting<Boolean> POOL_ENABLED = Setting.boolSetting("user_search.pool.enabled",
             settings -> Boolean.toString(PoolingSessionFactorySettings.BIND_DN.exists(settings)), Setting.Property.NodeScope);
 
@@ -36,6 +40,10 @@ public static Set<Setting<?>> getSettings() {
         settings.add(Setting.simpleString(AD_UPN_USER_SEARCH_FILTER_SETTING, Setting.Property.NodeScope));
         settings.add(Setting.simpleString(AD_DOWN_LEVEL_USER_SEARCH_FILTER_SETTING, Setting.Property.NodeScope));
         settings.add(Setting.simpleString(AD_USER_SEARCH_SCOPE_SETTING, Setting.Property.NodeScope));
+        settings.add(AD_LDAP_PORT_SETTING);
+        settings.add(AD_LDAPS_PORT_SETTING);
+        settings.add(AD_GC_LDAP_PORT_SETTING);
+        settings.add(AD_GC_LDAPS_PORT_SETTING);
         settings.add(POOL_ENABLED);
         settings.addAll(PoolingSessionFactorySettings.getSettings());
         return settings;

plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLConfigurationReloaderTests.java

@@ -128,7 +128,7 @@ public void testReloadingKeyStore() throws Exception {
             assertThat(aliases[0], is("key"));
         };
         final BiConsumer<X509ExtendedTrustManager, SSLConfiguration> trustManagerPostChecks = (updatedTrustManager, config) -> {
-            assertThat(trustedCount.get() - updatedTrustManager.getAcceptedIssuers().length, is(4));
+            assertThat(trustedCount.get() - updatedTrustManager.getAcceptedIssuers().length, is(5));
         };
         validateSSLConfigurationIsReloaded(settings, env, keyManagerPreChecks, trustManagerPreChecks, modifier, keyManagerPostChecks,
                 trustManagerPostChecks);
@@ -244,7 +244,7 @@ public void testReloadingTrustStore() throws Exception {
         };
 
         final BiConsumer<X509ExtendedTrustManager, SSLConfiguration> trustManagerPostChecks = (updatedTrustManager, config) -> {
-            assertThat(trustedCount.get() - updatedTrustManager.getAcceptedIssuers().length, is(5));
+            assertThat(trustedCount.get() - updatedTrustManager.getAcceptedIssuers().length, is(6));
         };
 
         validateTrustConfigurationIsReloaded(settings, env, trustManagerPreChecks, modifier, trustManagerPostChecks);

plugin/core/src/test/java/org/elasticsearch/xpack/core/ssl/SSLServiceTests.java

@@ -485,7 +485,7 @@ public void testReadCertificateInformation() throws Exception {
 
         final SSLService sslService = new SSLService(settings, env);
         final List<CertificateInfo> certificates = new ArrayList<>(sslService.getLoadedCertificates());
-        assertThat(certificates, iterableWithSize(7));
+        assertThat(certificates, iterableWithSize(8));
         Collections.sort(certificates,
                 Comparator.comparing((CertificateInfo c) -> c.alias() == null ? "" : c.alias()).thenComparing(CertificateInfo::path));
 
@@ -508,6 +508,15 @@ public void testReadCertificateInformation() throws Exception {
         assertThat(cert.expiry(), equalTo(DateTime.parse("2029-08-27T16:32:42Z")));
         assertThat(cert.hasPrivateKey(), equalTo(false));
 
+        cert = iterator.next();
+        assertThat(cert.alias(), equalTo("mykey"));
+        assertThat(cert.path(), equalTo(jksPath.toString()));
+        assertThat(cert.format(), equalTo("jks"));
+        assertThat(cert.serialNumber(), equalTo("3151a81eec8d4e34c56a8466a8510bcfbe63cc31"));
+        assertThat(cert.subjectDn(), equalTo("CN=samba4"));
+        assertThat(cert.expiry(), equalTo(DateTime.parse("2021-02-14T17:49:11.000Z")));
+        assertThat(cert.hasPrivateKey(), equalTo(false));
+
         cert = iterator.next();
         assertThat(cert.alias(), equalTo("openldap"));
         assertThat(cert.path(), equalTo(jksPath.toString()));

plugin/core/src/test/resources/org/elasticsearch/xpack/security/transport/ssl/certs/simple/testnode.jks

@@ -62,6 +62,8 @@ class ActiveDirectorySessionFactory extends PoolingSessionFactory {
     final DownLevelADAuthenticator downLevelADAuthenticator;
     final UpnADAuthenticator upnADAuthenticator;
 
+    private final int ldapPort;
+
     ActiveDirectorySessionFactory(RealmConfig config, SSLService sslService, ThreadPool threadPool) throws LDAPException {
         super(config, sslService, new ActiveDirectoryGroupsResolver(config.settings()),
                 ActiveDirectorySessionFactorySettings.POOL_ENABLED, () -> {
@@ -88,18 +90,24 @@ class ActiveDirectorySessionFactory extends PoolingSessionFactory {
                     + "] setting for active directory");
         }
         String domainDN = buildDnFromDomain(domainName);
+        ldapPort = ActiveDirectorySessionFactorySettings.AD_LDAP_PORT_SETTING.get(settings);
+        final int ldapsPort = ActiveDirectorySessionFactorySettings.AD_LDAPS_PORT_SETTING.get(settings);
+        final int gcLdapPort = ActiveDirectorySessionFactorySettings.AD_GC_LDAP_PORT_SETTING.get(settings);
+        final int gcLdapsPort = ActiveDirectorySessionFactorySettings.AD_GC_LDAPS_PORT_SETTING.get(settings);
+
         defaultADAuthenticator = new DefaultADAuthenticator(config, timeout, ignoreReferralErrors, logger, groupResolver,
                 metaDataResolver, domainDN, threadPool);
         downLevelADAuthenticator = new DownLevelADAuthenticator(config, timeout, ignoreReferralErrors, logger, groupResolver,
-                metaDataResolver, domainDN, sslService, threadPool);
+                metaDataResolver, domainDN, sslService, threadPool, ldapPort, ldapsPort, gcLdapPort, gcLdapsPort);
         upnADAuthenticator = new UpnADAuthenticator(config, timeout, ignoreReferralErrors, logger, groupResolver,
                 metaDataResolver, domainDN, threadPool);
 
     }
 
     @Override
     protected List<String> getDefaultLdapUrls(Settings settings) {
-        return Collections.singletonList("ldap://" + settings.get(ActiveDirectorySessionFactorySettings.AD_DOMAIN_NAME_SETTING) + ":389");
+        return Collections.singletonList("ldap://" + settings.get(ActiveDirectorySessionFactorySettings.AD_DOMAIN_NAME_SETTING) +
+                ":" + ldapPort);
     }
 
     @Override
@@ -361,16 +369,24 @@ static class DownLevelADAuthenticator extends ADAuthenticator {
         final Settings settings;
         final SSLService sslService;
         final RealmConfig config;
+        private final int ldapPort;
+        private final int ldapsPort;
+        private final int gcLdapPort;
+        private final int gcLdapsPort;
 
         DownLevelADAuthenticator(RealmConfig config, TimeValue timeout, boolean ignoreReferralErrors, Logger logger,
                 GroupsResolver groupsResolver, LdapMetaDataResolver metaDataResolver, String domainDN, SSLService sslService,
-                ThreadPool threadPool) {
+                ThreadPool threadPool, int ldapPort, int ldapsPort, int gcLdapPort, int gcLdapsPort) {
             super(config, timeout, ignoreReferralErrors, logger, groupsResolver, metaDataResolver, domainDN,
                     ActiveDirectorySessionFactorySettings.AD_DOWN_LEVEL_USER_SEARCH_FILTER_SETTING, DOWN_LEVEL_FILTER, threadPool);
             this.domainDN = domainDN;
             this.settings = config.settings();
             this.sslService = sslService;
             this.config = config;
+            this.ldapPort = ldapPort;
+            this.ldapsPort = ldapsPort;
+            this.gcLdapPort = gcLdapPort;
+            this.gcLdapsPort = gcLdapsPort;
         }
 
         @Override
@@ -400,7 +416,8 @@ void netBiosDomainNameToDn(LDAPInterface ldapInterface, String netBiosDomainName
                 if (cachedName != null) {
                     listener.onResponse(cachedName);
                 } else if (usingGlobalCatalog(ldapInterface) == false) {
-                    search(ldapInterface, domainDN, LdapSearchScope.SUB_TREE.scope(), filter, timeLimitSeconds, ignoreReferralErrors,
+                    search(ldapInterface, "CN=Configuration," + domainDN, LdapSearchScope.SUB_TREE.scope(), filter, timeLimitSeconds,
+                            ignoreReferralErrors,
                             ActionListener.wrap((results) -> handleSearchResults(results, netBiosDomainName, domainNameCache, listener),
                                     listener::onFailure),
                             "ncname");
@@ -417,16 +434,17 @@ void netBiosDomainNameToDn(LDAPInterface ldapInterface, String netBiosDomainName
                     final LDAPConnection finalLdapConnection = ldapConnection;
                     final LDAPConnection searchConnection = LdapUtils.privilegedConnect(
                             () -> new LDAPConnection(finalLdapConnection.getSocketFactory(), connectionOptions(config, sslService, logger),
-                                    finalLdapConnection.getConnectedAddress(), finalLdapConnection.getSSLSession() != null ? 636 : 389));
+                                    finalLdapConnection.getConnectedAddress(),
+                                    finalLdapConnection.getSSLSession() != null ? ldapsPort : ldapPort));
                     final byte[] passwordBytes = CharArrays.toUtf8Bytes(password.getChars());
                     final SimpleBindRequest bind = bindDN.isEmpty()
                             ? new SimpleBindRequest(username, passwordBytes)
                             : new SimpleBindRequest(bindDN, bindPassword);
                     LdapUtils.maybeForkThenBind(searchConnection, bind, threadPool, new ActionRunnable<String>(listener) {
                         @Override
                         protected void doRun() throws Exception {
-                            search(searchConnection, domainDN, LdapSearchScope.SUB_TREE.scope(), filter, timeLimitSeconds,
-                                    ignoreReferralErrors,
+                            search(searchConnection, "CN=Configuration," + domainDN, LdapSearchScope.SUB_TREE.scope(), filter,
+                                    timeLimitSeconds, ignoreReferralErrors,
                                     ActionListener.wrap(
                                             results -> {
                                                 IOUtils.close(searchConnection);
@@ -473,7 +491,7 @@ static void handleSearchResults(List<SearchResultEntry> results, String netBiosD
             }
         }
 
-        static boolean usingGlobalCatalog(LDAPInterface ldap) throws LDAPException {
+        boolean usingGlobalCatalog(LDAPInterface ldap) throws LDAPException {
             if (ldap instanceof LDAPConnection) {
                 return usingGlobalCatalog((LDAPConnection) ldap);
             } else {
@@ -490,8 +508,8 @@ static boolean usingGlobalCatalog(LDAPInterface ldap) throws LDAPException {
             }
         }
 
-        private static boolean usingGlobalCatalog(LDAPConnection ldapConnection) {
-            return ldapConnection.getConnectedPort() == 3268 || ldapConnection.getConnectedPort() == 3269;
+        private boolean usingGlobalCatalog(LDAPConnection ldapConnection) {
+            return ldapConnection.getConnectedPort() == gcLdapPort || ldapConnection.getConnectedPort() == gcLdapsPort;
         }
     }
 

plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java

@@ -13,7 +13,6 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
 import org.elasticsearch.env.TestEnvironment;
-import org.elasticsearch.test.junit.annotations.Network;
 import org.elasticsearch.threadpool.TestThreadPool;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.xpack.core.security.authc.RealmConfig;
@@ -28,12 +27,10 @@
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 
-import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
-import static org.hamcrest.Matchers.lessThan;
 
 public class LdapSessionFactoryTests extends LdapTestCase {
     private Settings globalSettings;

plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/LdapSessionFactoryTests.java

@@ -44,7 +44,7 @@
 
 import static org.elasticsearch.xpack.core.security.authc.ldap.support.SessionFactorySettings.HOSTNAME_VERIFICATION_SETTING;
 import static org.elasticsearch.xpack.core.security.authc.ldap.support.SessionFactorySettings.URLS_SETTING;
-//
+
 public abstract class LdapTestCase extends ESTestCase {
 
     private static final String USER_DN_TEMPLATES_SETTING_KEY = LdapSessionFactorySettings.USER_DN_TEMPLATES_SETTING.getKey();
@@ -120,8 +120,7 @@ public static Settings buildLdapSettings(String[] ldapUrl, String[] userTemplate
                 .put(SessionFactorySettings.TIMEOUT_TCP_CONNECTION_SETTING, TimeValue.timeValueSeconds(1L))
                 .put(SessionFactorySettings.IGNORE_REFERRAL_ERRORS_SETTING.getKey(), ignoreReferralErrors)
                 .put("group_search.base_dn", groupSearchBase)
-                .put("group_search.scope", scope)
-                .put("ssl.verification_mode", VerificationMode.CERTIFICATE);
+                .put("group_search.scope", scope);
         if (serverSetType != null) {
             builder.put(LdapLoadBalancingSettings.LOAD_BALANCE_SETTINGS + "." +
                             LdapLoadBalancingSettings.LOAD_BALANCE_TYPE_SETTING, serverSetType.toString());

plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/support/LdapTestCase.java

@@ -21,7 +21,7 @@
 import static org.hamcrest.Matchers.notNullValue;
 
 @SuppressWarnings("unchecked")
-            public class SearchGroupsResolverTests extends GroupsResolverTestCase {
+public class SearchGroupsResolverTests extends GroupsResolverTestCase {
 
     private static final String BRUCE_BANNER_DN = "uid=hulk,ou=people,dc=oldap,dc=test,dc=elasticsearch,dc=com";
 

plugin/security/src/test/resources/org/elasticsearch/xpack/security/authc/ldap/support/ADtrust.jks

@@ -1,14 +1,19 @@
+Project smbFixtureProject = xpackProject("test:smb-fixture")
+evaluationDependsOn(smbFixtureProject.path)
+
+apply plugin: 'elasticsearch.vagrantsupport'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
   testCompile project(xpackModule('security'))
   testCompile project(path: xpackModule('security'), configuration: 'testArtifacts')
 }
 
-// add test resources from security, so certificate tool tests can use example certs
+// add test resources from security, so tests can use example certs
 sourceSets.test.resources.srcDirs(project(xpackModule('security')).sourceSets.test.resources.srcDirs)
+compileTestJava.options.compilerArgs << "-Xlint:-deprecation,-rawtypes,-serial,-try,-unchecked"
 
-// we have to repeate these patterns because the security test resources are effectively in the src of this project
+// we have to repeat these patterns because the security test resources are effectively in the src of this project
 forbiddenPatterns {
   exclude '**/*.key'
   exclude '**/*.p12'
@@ -21,7 +26,22 @@ test {
    * other if we allow them to set the number of available processors as it's set-once in Netty.
    */
   systemProperty 'es.set.netty.runtime.available.processors', 'false'
+  include '**/*IT.class'
+  include '**/*Tests.class'
 }
 
 // these are just tests, no need to audit
 thirdPartyAudit.enabled = false
+
+task smbFixture {
+  dependsOn "vagrantCheckVersion", "virtualboxCheckVersion", smbFixtureProject.up
+}
+
+if (project.rootProject.vagrantSupported) {
+  if (project.hasProperty('useExternalAD') == false) {
+    test.dependsOn smbFixture
+    test.finalizedBy smbFixtureProject.halt
+  }
+} else {
+  test.enabled = false
+}