Address test failures for SmokeTestWatcherWithSecurityIT (#42092)

* Address test failures for SmokeTestWatcherWithSecurityIT

There are likely multiple root causes to the seemingly random failures
generated by SmokeTestWatcherWithSecurityIT. This commit un-mutes this
this test, address one known cause and adds debug logging for this test.

The known root cause for one failure is that we can have a Watch running
that is reading data from an index. Before we stop Watcher we delete that
index. If Watcher happens to execute after deletion of the index but before
the stop of Watcher the test can fail. The fix here is to simply move the
index deletion after the stop of Watcher.

Related #35361
Related #30777
Related #33291
Related #29893

Author: jakelandis

x-pack/qa/smoke-test-watcher-with-security/build.gradle

@@ -24,6 +24,8 @@ integTestCluster {
   setting 'xpack.notification.email.account._email.smtp.user', '_user'
   keystoreSetting 'xpack.notification.email.account._email.smtp.secure_password', '_passwd'
   setting 'xpack.license.self_generated.type', 'trial'
+  setting 'logger.org.elasticsearch.xpack.watcher', 'debug'
+  setting 'logger.org.elasticsearch.xpack.core.watcher', 'debug'
   extraConfigFile 'roles.yml', 'roles.yml'
   setupCommand 'setupTestAdminUser',
                'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'

x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.smoketest;
 
 import org.apache.http.util.EntityUtils;
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
 import org.elasticsearch.common.Strings;
@@ -32,7 +31,6 @@
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35361")
 public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
 
     private static final String TEST_ADMIN_USERNAME = "test_admin";
@@ -91,7 +89,6 @@ public void startWatcher() throws Exception {
 
     @After
     public void stopWatcher() throws Exception {
-        adminClient().performRequest(new Request("DELETE", "/my_test_index"));
 
         assertBusy(() -> {
             try {
@@ -119,6 +116,8 @@ public void stopWatcher() throws Exception {
                 throw new AssertionError(e);
             }
         });
+
+        adminClient().performRequest(new Request("DELETE", "/my_test_index"));
     }
 
     @Override