Ensure a unique WatchId for test in SmokeTestWatcherWithSecurityIT

This commit un-mutes SmokeTestWatcherWithSecurityIT by ensuring that between test invocations
a new WatchId is used. It is believed that since the same WatchId is used between tests,
and that Watch is never deleted, the test failures may be due to a race condition between
the current Watch execution, the new Watch (with the same ID), and the busy wait for the
Watcher history document.

This race condtion can be avoided by simply using a new WatchId and removing the Watch
between test invocations.

This commit also adds a minor change (XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) that is
needed since the test was muted.

Fixes elastic#35361

Author: jakelandis

x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java

@@ -32,16 +32,16 @@
 import static org.hamcrest.Matchers.hasEntry;
 import static org.hamcrest.Matchers.is;
 
-@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/35361")
 public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase {
 
     private static final String TEST_ADMIN_USERNAME = "test_admin";
     private static final String TEST_ADMIN_PASSWORD = "x-pack-test-password";
 
-    private String watchId = randomAlphaOfLength(20);
+    private String watchId = null;
 
     @Before
     public void startWatcher() throws Exception {
+        watchId = randomAlphaOfLength(20);
         Request createAllowedDoc = new Request("PUT", "/my_test_index/_doc/1");
         createAllowedDoc.setJsonEntity("{ \"value\" : \"15\" }");
         createAllowedDoc.addParameter("refresh", "true");
@@ -83,7 +83,7 @@ public void startWatcher() throws Exception {
         });
 
         assertBusy(() -> {
-            for (String template : XPackRestTestConstants.TEMPLATE_NAMES) {
+            for (String template : XPackRestTestConstants.TEMPLATE_NAMES_NO_ILM) {
                 assertOK(adminClient().performRequest(new Request("HEAD", "_template/" + template)));
             }
         });
@@ -92,6 +92,10 @@ public void startWatcher() throws Exception {
     @After
     public void stopWatcher() throws Exception {
         adminClient().performRequest(new Request("DELETE", "/my_test_index"));
+        Response response = adminClient().performRequest(new Request("GET", "_watcher/watch/" + watchId));
+        if (response.getStatusLine().getStatusCode() == 200) {
+            adminClient().performRequest(new Request("DELETE", "_watcher/watch/" + watchId));
+        }
 
         assertBusy(() -> {
             try {

x-pack/qa/src/main/java/org/elasticsearch/xpack/test/rest/XPackRestTestConstants.java

@@ -11,13 +11,9 @@ public final class XPackRestTestConstants {
 
     // Watcher constants:
     public static final String INDEX_TEMPLATE_VERSION = "9";
-    public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION;
     public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION;
     public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches";
     public static final String WATCHES_TEMPLATE_NAME = ".watches";
-    public static final String[] TEMPLATE_NAMES = new String[] {
-        HISTORY_TEMPLATE_NAME, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME
-    };
     public static final String[] TEMPLATE_NAMES_NO_ILM = new String[] {
         HISTORY_TEMPLATE_NAME_NO_ILM, TRIGGERED_TEMPLATE_NAME, WATCHES_TEMPLATE_NAME
     };