Add more trace logging when installing monitor watches and (elastic#68752)

unmute TransportMonitoringMigrateAlertsActionTests#testLocalAlertsRemoval and
TransportMonitoringMigrateAlertsActionTests#testRepeatedLocalAlertsRemoval tests

Somehow during these tests the monitor watches are not installed. Both
tests use the local exporter and this exporter only installs the watches
under specific conditions via the elected master node. I suspect the
conditions are never met. The http exporter is more relaxed when attempting
to install monitor watches and the tests using the http exporter seem
not to be prone by the fact that tests fail because monitor watches have
not been installed.

Relates to elastic#66586

Author: martijnvg

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

@@ -430,9 +430,20 @@ private void setupClusterAlertsTasks(ClusterState clusterState, boolean clusterS
                 if (watches != null && watches.allPrimaryShardsActive() == false) {
                     logger.trace("cannot manage cluster alerts because [.watches] index is not allocated");
                 } else if ((watches == null || indexExists) && watcherSetup.compareAndSet(false, true)) {
+                    logger.trace("installing monitoring watches");
                     getClusterAlertsInstallationAsyncActions(indexExists, asyncActions, pendingResponses);
+                } else {
+                    logger.trace("skipping installing monitoring watches, watches=[{}], indexExists=[{}], watcherSetup=[{}]",
+                        watches, indexExists, watcherSetup.get());
                 }
+            } else {
+                logger.trace("watches shouldn't be setup, because state=[{}] and clusterStateChange=[{}]", state.get(), clusterStateChange);
             }
+        } else {
+            logger.trace("watches can't be used, because xpack.watcher.enabled=[{}] and " +
+                    "xpack.monitoring.exporters._local.cluster_alerts.management.enabled=[{}]",
+                XPackSettings.WATCHER_ENABLED.get(config.settings()),
+                CLUSTER_ALERTS_MANAGEMENT_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings()));
         }
     }
 
@@ -581,6 +592,7 @@ private void getClusterAlertsInstallationAsyncActions(final boolean indexExists,
                                                                new ResponseActionListener<>("watch", uniqueWatchId, pendingResponses)));
                 }
             } else if (addWatch) {
+                logger.trace("adding monitoring watch [{}]", uniqueWatchId);
                 asyncActions.add(() -> putWatch(watcher, watchId, uniqueWatchId, pendingResponses));
             }
         }

x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringMigrateAlertsActionTests.java

@@ -35,6 +35,7 @@
 import org.elasticsearch.test.http.MockRequest;
 import org.elasticsearch.test.http.MockResponse;
 import org.elasticsearch.test.http.MockWebServer;
+import org.elasticsearch.test.junit.annotations.TestLogging;
 import org.elasticsearch.xpack.core.XPackSettings;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsAction;
 import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsRequest;
@@ -124,7 +125,9 @@ private void stopMonitoring() {
         ));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/66586")
+    @TestLogging(
+        value = "org.elasticsearch.xpack.monitoring.exporter.local:trace",
+        reason = "to ensure we log local exporter on trace level")
     public void testLocalAlertsRemoval() throws Exception {
         try {
             // start monitoring service
@@ -159,6 +162,9 @@ public void testLocalAlertsRemoval() throws Exception {
         }
     }
 
+    @TestLogging(
+        value = "org.elasticsearch.xpack.monitoring.exporter.local:trace",
+        reason = "to ensure we log local exporter on trace level")
     public void testRepeatedLocalAlertsRemoval() throws Exception {
         try {
             // start monitoring service