Add extra steps to ensure that watch indices are created before test starts (#66416) (#66422)

This PR updates the initialization logic in TransportMonitoringMigrateAlertsActionTests to wait
for the watch indices separately from the rest of the initialization logic in order to allow for the
.watches index to be available before the test begins.

Author: jbaiera

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction;
 import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest;
 import org.elasticsearch.xpack.monitoring.LocalStateMonitoring;
+import org.elasticsearch.xpack.core.watcher.watch.Watch;
 import org.elasticsearch.xpack.monitoring.Monitoring;
 import org.elasticsearch.xpack.monitoring.MonitoringService;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
@@ -135,12 +136,7 @@ public void testLocalAlertsRemoval() throws Exception {
             assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings));
 
             // ensure resources exist
-            assertBusy(() -> {
-                assertThat(indexExists(".monitoring-*"), is(true));
-                ensureYellowAndNoInitializingShards(".monitoring-*");
-                checkMonitoringTemplates();
-                assertWatchesExist(true);
-            });
+            ensureInitialLocalResources();
 
             // call migration api
             MonitoringMigrateAlertsResponse response = client().execute(MonitoringMigrateAlertsAction.INSTANCE,
@@ -174,12 +170,7 @@ public void testRepeatedLocalAlertsRemoval() throws Exception {
             assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(exporterSettings));
 
             // ensure resources exist
-            assertBusy(() -> {
-                assertThat(indexExists(".monitoring-*"), is(true));
-                ensureYellowAndNoInitializingShards(".monitoring-*");
-                checkMonitoringTemplates();
-                assertWatchesExist(true);
-            });
+            ensureInitialLocalResources();
 
             // call migration api
             MonitoringMigrateAlertsResponse response = client().execute(MonitoringMigrateAlertsAction.INSTANCE,
@@ -479,6 +470,7 @@ public void testRemoteAlertsRemoteDisallowsWatcher() throws Exception {
     }
 
     private void ensureInitialLocalResources() throws Exception {
+        waitForWatcherIndices();
         assertBusy(() -> {
             assertThat(indexExists(".monitoring-*"), is(true));
             ensureYellowAndNoInitializingShards(".monitoring-*");
@@ -595,4 +587,9 @@ private void assertMonitorWatches(final MockWebServer webServer, final boolean r
             }
         }
     }
+
+    protected void waitForWatcherIndices() throws Exception {
+        awaitIndexExists(Watch.INDEX);
+        assertBusy(() -> ensureYellowAndNoInitializingShards(Watch.INDEX));
+    }
 }

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java

@@ -200,7 +200,7 @@ protected void waitForMonitoringIndices() throws Exception {
         assertBusy(this::ensureMonitoringIndicesYellow);
     }
 
-    private void awaitIndexExists(final String index) throws Exception {
+    protected void awaitIndexExists(final String index) throws Exception {
         assertBusy(() -> assertIndicesExists(index), 30, TimeUnit.SECONDS);
     }