From 62aa8ecab53da5f806f286d78d36ec961d1ab940 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Wed, 16 Sep 2020 14:18:04 -0400 Subject: [PATCH 1/6] WIP --- .../xpack/monitoring/Monitoring.java | 3 + .../xpack/monitoring/exporter/Exporters.java | 5 +- .../exporter/http/HttpExporter.java | 4 +- .../exporter/local/LocalExporter.java | 5 +- .../LocalExporterResourceIntegTests.java | 91 ++++++++++++++++++- 5 files changed, 100 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java index fcf7299904e06..9595427c59e54 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java @@ -77,6 +77,9 @@ public class Monitoring extends Plugin implements ActionPlugin, ReloadablePlugin public static final Setting CLEAN_WATCHER_HISTORY = boolSetting("xpack.watcher.history.cleaner_service.enabled", true, Setting.Property.Dynamic, Setting.Property.NodeScope, Setting.Property.Deprecated); + public static final Setting MIGRATION_DECOMMISSION_ALERTS = boolSetting("xpack.monitoring.migration.decommission_alerts", + false, Setting.Property.Dynamic, Setting.Property.NodeScope); + protected final Settings settings; private Exporters exporters; diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java index 7cff593def80a..7e1df36b4ba0b 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/Exporters.java @@ -23,6 +23,7 @@ import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.xpack.core.ssl.SSLService; +import org.elasticsearch.xpack.monitoring.Monitoring; import org.elasticsearch.xpack.monitoring.exporter.http.HttpExporter; import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringDoc; import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporter; @@ -62,7 +63,9 @@ public Exporters(Settings settings, Map factories, final List> dynamicSettings = getSettings().stream().filter(Setting::isDynamic).collect(Collectors.toList()); - clusterService.getClusterSettings().addSettingsUpdateConsumer(this::setExportersSetting, dynamicSettings); + final List> updateSettings = new ArrayList>(dynamicSettings); + updateSettings.add(Monitoring.MIGRATION_DECOMMISSION_ALERTS); + clusterService.getClusterSettings().addSettingsUpdateConsumer(this::setExportersSetting, updateSettings); HttpExporter.registerSettingValidators(clusterService, sslService); // this ensures that logging is happening by adding an empty consumer per affix setting for (Setting.AffixSetting affixSetting : dynamicSettings) { diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java index d49cc2973823e..875c3acb482be 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java @@ -43,10 +43,12 @@ import org.elasticsearch.xpack.core.ssl.SSLConfiguration; import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings; import org.elasticsearch.xpack.core.ssl.SSLService; +import org.elasticsearch.xpack.monitoring.Monitoring; import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil; import org.elasticsearch.xpack.monitoring.exporter.ExportBulk; import org.elasticsearch.xpack.monitoring.exporter.Exporter; +import javax.management.monitor.Monitor; import javax.net.ssl.SSLContext; import java.util.ArrayList; import java.util.Arrays; @@ -837,7 +839,7 @@ private static void configureClusterAlertsResources(final Config config, final S // add a resource per watch for (final String watchId : ClusterAlertsUtil.WATCH_IDS) { - final boolean blacklisted = blacklist.contains(watchId); + final boolean blacklisted = blacklist.contains(watchId) || Monitoring.MIGRATION_DECOMMISSION_ALERTS.get(config.settings()); // lazily load the cluster state to fetch the cluster UUID once it's loaded final Supplier uniqueWatchId = () -> ClusterAlertsUtil.createUniqueWatchId(clusterService, watchId); final Supplier watch = blacklisted ? null : () -> ClusterAlertsUtil.loadWatch(clusterService, watchId); diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 131bda16cd59f..41fe43110cbca 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -51,6 +51,7 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchResponse; import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction; import org.elasticsearch.xpack.core.watcher.watch.Watch; +import org.elasticsearch.xpack.monitoring.Monitoring; import org.elasticsearch.xpack.monitoring.cleaner.CleanerService; import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil; import org.elasticsearch.xpack.monitoring.exporter.ExportBulk; @@ -106,6 +107,7 @@ public class LocalExporter extends Exporter implements ClusterStateListener, Cle private final boolean useIngest; private final DateFormatter dateTimeFormatter; private final List clusterAlertBlacklist; + private final boolean decommissionClusterAlerts; private final AtomicReference state = new AtomicReference<>(State.INITIALIZED); private final AtomicBoolean installingSomething = new AtomicBoolean(false); @@ -121,6 +123,7 @@ public LocalExporter(Exporter.Config config, Client client, CleanerService clean this.licenseState = config.licenseState(); this.useIngest = USE_INGEST_PIPELINE_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings()); this.clusterAlertBlacklist = ClusterAlertsUtil.getClusterAlertsBlacklist(config); + this.decommissionClusterAlerts = Monitoring.MIGRATION_DECOMMISSION_ALERTS.get(config.settings()); this.cleanerService = cleanerService; this.dateTimeFormatter = dateTimeFormatter(config); // if additional listeners are added here, adjust LocalExporterTests#testLocalExporterRemovesListenersOnClose accordingly @@ -462,7 +465,7 @@ private void getClusterAlertsInstallationAsyncActions(final boolean indexExists, for (final String watchId : ClusterAlertsUtil.WATCH_IDS) { final String uniqueWatchId = ClusterAlertsUtil.createUniqueWatchId(clusterService, watchId); - final boolean addWatch = canAddWatches && clusterAlertBlacklist.contains(watchId) == false; + final boolean addWatch = canAddWatches && clusterAlertBlacklist.contains(watchId) == false && decommissionClusterAlerts == false; // we aren't sure if no watches exist yet, so add them if (indexExists) { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java index f18228b9ec005..fab3247d6ea62 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java @@ -7,25 +7,40 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ObjectPath; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.ingest.PipelineConfiguration; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest; +import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.core.monitoring.MonitoredSystem; import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils; +import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction; +import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction; +import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest; +import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction; import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil; import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; +import static org.elasticsearch.xpack.core.ClientHelper.MONITORING_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; +import static org.hamcrest.Matchers.*; @ESIntegTestCase.ClusterScope(scope = TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) @@ -102,6 +117,7 @@ private void putResources(final Integer version) throws Exception { putTemplate(version); putPipelines(version); + putWatches(version); } private void putTemplate(final Integer version) throws Exception { @@ -150,6 +166,56 @@ private BytesReference replaceablePipeline(final Integer version) { } } + /** + * Create a cluster alert that does nothing. + * @param version Version to add to the watch, if any + */ + private void putWatches(final Integer version) throws Exception { + for (final String watchId : ClusterAlertsUtil.WATCH_IDS) { + final String uniqueWatchId = ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId); + logger.info("Making a watch! " + uniqueWatchId); + final BytesReference watch = generateWatchSource(uniqueWatchId, clusterService().state().metadata().clusterUUID(), version); + client().execute(PutWatchAction.INSTANCE, new PutWatchRequest(uniqueWatchId, watch, XContentType.JSON)) + .actionGet(); + } + } + + /** + * Generates a basic watch that loosely represents a monitoring cluster alert but ultimately does nothing. + */ + private static BytesReference generateWatchSource(final String id, final String clusterUUID, final Integer version) throws Exception { + final XContentBuilder builder = jsonBuilder().startObject(); + builder + .startObject("metadata") + .startObject("xpack") + .field("cluster_uuid", clusterUUID); + if(version != null) { + builder.field("version_created", version); + } + builder + .field("watch", id) + .endObject() + .endObject() + .startObject("trigger") + .startObject("schedule") + .field("interval", "30m") + .endObject() + .endObject() + .startObject("input") + .startObject("simple") + .field("ignore", "ignore") + .endObject() + .endObject() + .startObject("condition") + .startObject("never") + .endObject() + .endObject() + .startObject("actions") + .endObject(); + + return BytesReference.bytes(builder.endObject()); + } + private Integer oldVersion() { final int minimumVersion = Math.min(ClusterAlertsUtil.LAST_UPDATED_VERSION, MonitoringTemplateUtils.LAST_UPDATED_VERSION); @@ -179,6 +245,20 @@ private void assertPipelinesExist() { } } + private void assertWatchesExist() { + String clusterUUID = clusterService().state().getMetadata().clusterUUID(); + SearchSourceBuilder searchSource = SearchSourceBuilder.searchSource() + .query(QueryBuilders.matchQuery("metadata.xpack.cluster_uuid", clusterService().state().getMetadata().clusterUUID())); + Set watchIds = new HashSet<>(Arrays.asList(ClusterAlertsUtil.WATCH_IDS)); + for (SearchHit hit : client().prepareSearch(".watches").setSource(searchSource).get().getHits().getHits()) { + String watchId = ObjectPath.eval("metadata.xpack.watch", hit.getSourceAsMap()); + assertTrue(watchId.startsWith(clusterUUID)); + assertTrue(watchId.length() > clusterUUID.length() + 1); + watchId = watchId.substring(clusterUUID.length() + 1); + assertThat("found unexpected watch id", watchIds, contains(watchId)); + } + } + private void assertResourcesExist() throws Exception { createResources(); @@ -187,6 +267,7 @@ private void assertResourcesExist() throws Exception { assertBusy(() -> { assertTemplatesExist(); assertPipelinesExist(); + assertWatchesExist(); }); } From f3c04deff3d784509bb2a1e790ae4bbce872dcd9 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Thu, 17 Sep 2020 16:41:01 -0400 Subject: [PATCH 2/6] Actually test creation and updating of watches in local monitoring. This will allow us to test if the exporters actually pull the watches back down when they aren't in use. --- .../LocalExporterResourceIntegTests.java | 68 ++++++++---- .../test/MockClusterAlertScriptEngine.java | 101 ++++++++++++++++++ .../test/MonitoringIntegTestCase.java | 6 +- 3 files changed, 149 insertions(+), 26 deletions(-) create mode 100644 x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockClusterAlertScriptEngine.java diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java index fab3247d6ea62..3c95af99964f1 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java @@ -5,51 +5,55 @@ */ package org.elasticsearch.xpack.monitoring.exporter.local; +import org.elasticsearch.Version; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ObjectPath; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.ingest.PipelineConfiguration; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.protocol.xpack.watcher.DeleteWatchRequest; import org.elasticsearch.protocol.xpack.watcher.PutWatchRequest; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.core.monitoring.MonitoredSystem; import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils; -import org.elasticsearch.xpack.core.watcher.transport.actions.delete.DeleteWatchAction; -import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchAction; -import org.elasticsearch.xpack.core.watcher.transport.actions.get.GetWatchRequest; import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchAction; import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; import java.util.HashSet; -import java.util.Map; import java.util.Set; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.ESIntegTestCase.Scope.TEST; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.xpack.core.ClientHelper.MONITORING_ORIGIN; -import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; -import static org.hamcrest.Matchers.*; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; @ESIntegTestCase.ClusterScope(scope = TEST, numDataNodes = 1, numClientNodes = 0, supportsDedicatedMasters = false) public class LocalExporterResourceIntegTests extends LocalExporterIntegTestCase { - public LocalExporterResourceIntegTests() throws Exception { + public LocalExporterResourceIntegTests() { super(); } + @Override + protected Settings nodeSettings(int nodeOrdinal) { + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put("xpack.license.self_generated.type", "trial") + .build(); + } + private final MonitoredSystem system = randomFrom(MonitoredSystem.values()); public void testCreateWhenResourcesNeedToBeAddedOrUpdated() throws Exception { @@ -71,6 +75,16 @@ public void testCreateWhenResourcesShouldNotBeReplaced() throws Exception { assertPipelinesNotUpdated(); } + @Override + protected Settings localExporterSettings() { + // Override the settings for local exporters created in this test, make sure watcher is enabled so we can test + // cluster alert creation and decommissioning + return Settings.builder() + .put(super.localExporterSettings()) + .put("xpack.monitoring.exporters." + exporterName + ".cluster_alerts.management.enabled", true) + .build(); + } + private void createResources() throws Exception { // wait until the cluster is ready (this is done at the "Exporters" level) // this is not a busy assertion because it's checked earlier @@ -127,13 +141,13 @@ private void putTemplate(final Integer version) throws Exception { assertAcked(client().admin().indices().preparePutTemplate(templateName).setSource(source, XContentType.JSON).get()); } - private void putPipelines(final Integer version) throws Exception { + private void putPipelines(final Integer version) { for (final String pipelineId : MonitoringTemplateUtils.PIPELINE_IDS) { putPipeline(MonitoringTemplateUtils.pipelineName(pipelineId), version); } } - private void putPipeline(final String pipelineName, final Integer version) throws Exception { + private void putPipeline(final String pipelineName, final Integer version) { assertAcked(client().admin().cluster().preparePutPipeline(pipelineName, replaceablePipeline(version), XContentType.JSON).get()); } @@ -173,8 +187,7 @@ private BytesReference replaceablePipeline(final Integer version) { private void putWatches(final Integer version) throws Exception { for (final String watchId : ClusterAlertsUtil.WATCH_IDS) { final String uniqueWatchId = ClusterAlertsUtil.createUniqueWatchId(clusterService(), watchId); - logger.info("Making a watch! " + uniqueWatchId); - final BytesReference watch = generateWatchSource(uniqueWatchId, clusterService().state().metadata().clusterUUID(), version); + final BytesReference watch = generateWatchSource(watchId, clusterService().state().metadata().clusterUUID(), version); client().execute(PutWatchAction.INSTANCE, new PutWatchRequest(uniqueWatchId, watch, XContentType.JSON)) .actionGet(); } @@ -190,7 +203,7 @@ private static BytesReference generateWatchSource(final String id, final String .startObject("xpack") .field("cluster_uuid", clusterUUID); if(version != null) { - builder.field("version_created", version); + builder.field("version_created", Integer.toString(version)); } builder .field("watch", id) @@ -246,16 +259,27 @@ private void assertPipelinesExist() { } private void assertWatchesExist() { + // Check if watches index exists + if (client().admin().indices().prepareGetIndex().addIndices(".watches").get().getIndices().length == 0) { + fail("Expected [.watches] index with cluster alerts present, but no [.watches] index was found"); + } + String clusterUUID = clusterService().state().getMetadata().clusterUUID(); SearchSourceBuilder searchSource = SearchSourceBuilder.searchSource() - .query(QueryBuilders.matchQuery("metadata.xpack.cluster_uuid", clusterService().state().getMetadata().clusterUUID())); + .query(QueryBuilders.matchQuery("metadata.xpack.cluster_uuid", clusterUUID)); Set watchIds = new HashSet<>(Arrays.asList(ClusterAlertsUtil.WATCH_IDS)); for (SearchHit hit : client().prepareSearch(".watches").setSource(searchSource).get().getHits().getHits()) { String watchId = ObjectPath.eval("metadata.xpack.watch", hit.getSourceAsMap()); - assertTrue(watchId.startsWith(clusterUUID)); - assertTrue(watchId.length() > clusterUUID.length() + 1); - watchId = watchId.substring(clusterUUID.length() + 1); - assertThat("found unexpected watch id", watchIds, contains(watchId)); + assertNotNull("Missing watch ID", watchId); + assertTrue("found unexpected watch id", watchIds.contains(watchId)); + + String version = ObjectPath.eval("metadata.xpack.version_created", hit.getSourceAsMap()); + assertNotNull("Missing version from returned watch [" + watchId + "]", version); + assertTrue(Version.fromId(Integer.parseInt(version)).onOrAfter(Version.fromId(ClusterAlertsUtil.LAST_UPDATED_VERSION))); + + String uuid = ObjectPath.eval("metadata.xpack.cluster_uuid", hit.getSourceAsMap()); + assertNotNull("Missing cluster uuid", uuid); + assertEquals(clusterUUID, uuid); } } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockClusterAlertScriptEngine.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockClusterAlertScriptEngine.java new file mode 100644 index 0000000000000..9b5ad1a15dc3e --- /dev/null +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockClusterAlertScriptEngine.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.monitoring.test; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.MockDeterministicScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.xpack.core.monitoring.test.MockPainlessScriptEngine; +import org.elasticsearch.xpack.core.watcher.execution.WatchExecutionContext; +import org.elasticsearch.xpack.core.watcher.watch.Payload; +import org.elasticsearch.xpack.watcher.condition.WatcherConditionScript; +import org.elasticsearch.xpack.watcher.transform.script.WatcherTransformScript; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * A mock scripting engine that allows for scripts used in the watcher cluster alert definitions to be parsed + * for the purposes of testing their creation and decommissioning. Their accuracy as watch definitions should + * be verified via some other avenue. + */ +public class MockClusterAlertScriptEngine extends MockPainlessScriptEngine { + + /** + * The plugin that creates this mock script engine. Overrides the original mock engine to inject this + * implementation instead of the parent class. + */ + public static class TestPlugin extends MockPainlessScriptEngine.TestPlugin { + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new MockClusterAlertScriptEngine(); + } + } + + @Override + public T compile(String name, String script, ScriptContext context, Map options) { + // Conditions - Default to false evaluation in testing + if (context.instanceClazz.equals(WatcherConditionScript.class)) { + return context.factoryClazz.cast(new MockWatcherConditionScript(MockDeterministicScript.asDeterministic(p -> false))); + } + + // Transform - Return empty Map from the transform function + if (context.instanceClazz.equals(WatcherTransformScript.class)) { + return context.factoryClazz.cast(new MockWatcherTransformScript(MockDeterministicScript.asDeterministic( + p -> new HashMap()))); + } + + // We want to just add an allowance for watcher scripts, and to delegate everything else to the parent class. + return super.compile(name, script, context, options); + } + + /** + * A mock watcher condition script that executes a given function instead of whatever the source provided was. + */ + static class MockWatcherConditionScript implements WatcherConditionScript.Factory { + + private final MockDeterministicScript script; + + public MockWatcherConditionScript(MockDeterministicScript script) { + this.script = script; + } + + @Override + public WatcherConditionScript newInstance(Map params, WatchExecutionContext watcherContext) { + return new WatcherConditionScript(params, watcherContext) { + @Override + public boolean execute() { + return (boolean) script.apply(getParams()); + } + }; + } + } + + /** + * A mock watcher transformation script that performs a given function instead of whatever the source provided was. + */ + static class MockWatcherTransformScript implements WatcherTransformScript.Factory { + + private final MockDeterministicScript script; + + public MockWatcherTransformScript(MockDeterministicScript script) { + this.script = script; + } + + @Override + public WatcherTransformScript newInstance(Map params, WatchExecutionContext watcherContext, Payload payload) { + return new WatcherTransformScript(params, watcherContext, payload) { + @Override + public Object execute() { + return script.apply(getParams()); + } + }; + } + } +} diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java index 2a23ee35bc130..fb81519982559 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java @@ -70,7 +70,7 @@ protected Collection> getMockPlugins() { @Override protected Collection> nodePlugins() { - return Arrays.asList(LocalStateMonitoring.class, MockPainlessScriptEngine.TestPlugin.class, + return Arrays.asList(LocalStateMonitoring.class, MockClusterAlertScriptEngine.TestPlugin.class, MockIngestPlugin.class, CommonAnalysisPlugin.class); } @@ -178,9 +178,7 @@ protected void waitForMonitoringIndices() throws Exception { } private void awaitIndexExists(final String index) throws Exception { - assertBusy(() -> { - assertIndicesExists(index); - }, 30, TimeUnit.SECONDS); + assertBusy(() -> assertIndicesExists(index), 30, TimeUnit.SECONDS); } private void assertIndicesExists(String... indices) { From f5c399dc4f8bc62948e515b759b57f02649c657d Mon Sep 17 00:00:00 2001 From: James Baiera Date: Fri, 18 Sep 2020 16:57:01 -0400 Subject: [PATCH 3/6] Add test cases that ensure cluster alerts are torn down when decommissioning setting is present. --- .../exporter/local/LocalExporter.java | 4 +- .../http/HttpExporterResourceTests.java | 55 ++++++++++++++++++- .../local/LocalExporterIntegTestCase.java | 18 ++++-- .../LocalExporterResourceIntegTests.java | 48 +++++++++++++++- 4 files changed, 117 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java index 41fe43110cbca..2e167164534e2 100644 --- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java +++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java @@ -162,8 +162,10 @@ public void licenseStateChanged() { boolean isExporterReady() { // forces the setup to occur if it hasn't already final boolean running = resolveBulk(clusterService.state(), false) != null; + // Report on watcher readiness + boolean alertsProcessed = !canUseWatcher() || watcherSetup.get(); - return running && installingSomething.get() == false; + return running && installingSomething.get() == false && alertsProcessed; } @Override diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java index 35da399692e24..d93d5a7a1df5c 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java @@ -102,7 +102,11 @@ public void setupResources() { } public void awaitCheckAndPublish(final Boolean expected) { - resources.checkAndPublish(client, listener); + awaitCheckAndPublish(resources, expected); + } + + public void awaitCheckAndPublish(HttpResource resource, final Boolean expected) { + resource.checkAndPublish(client, listener); verifyListener(expected); } @@ -484,6 +488,55 @@ public void testWatchPublishBlocksAfterSuccessfulWatcherCheck() { verifyNoMoreInteractions(client); } + public void testWatchRemovalOnDecomissionAlerts() { + final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES); + final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates; + final int successfulGetPipelines = randomIntBetween(0, EXPECTED_PIPELINES); + final int unsuccessfulGetPipelines = EXPECTED_PIPELINES - successfulGetPipelines; + final Exception exception = failurePutException(); + + whenValidVersionResponse(); + whenGetTemplates(successfulGetTemplates, unsuccessfulGetTemplates); + whenSuccessfulPutTemplates(unsuccessfulGetTemplates); + whenGetPipelines(successfulGetPipelines, unsuccessfulGetPipelines); + whenSuccessfulPutPipelines(unsuccessfulGetPipelines); + // license needs to be valid, otherwise we'll do DELETEs, which are tested earlier + whenWatcherCanBeUsed(true); + + // a number of watches are mocked as present + final int existingWatches = randomIntBetween(0, EXPECTED_WATCHES); + + // For completeness's sake. GET/PUT watches wont be called by the resources. Instead it tries to DELETE the watches ignoring them not existing. + whenGetWatches(existingWatches, EXPECTED_WATCHES - existingWatches); + whenPerformRequestAsyncWith(client, new RequestMatcher(is("PUT"), startsWith("/_watcher/watch/")), exception); + whenPerformRequestAsyncWith(client, new RequestMatcher(is("DELETE"), startsWith("/_watcher/watch/")), + successfulDeleteResponses(EXPECTED_WATCHES)); + + // Create resources that are configured to remove all watches + Settings removalExporterSettings = Settings.builder() + .put(exporterSettings) + .put("xpack.monitoring.migration.decommission_alerts", true) + .build(); + MultiHttpResource overrideResource = HttpExporter.createResources( + new Exporter.Config("_http", "http", removalExporterSettings, clusterService, licenseState)); + + assertTrue(overrideResource.isDirty()); + awaitCheckAndPublish(overrideResource, true); + // Should proceed + assertFalse(overrideResource.isDirty()); + + verifyVersionCheck(); + verifyGetTemplates(EXPECTED_TEMPLATES); + verifyPutTemplates(unsuccessfulGetTemplates); + verifyGetPipelines(EXPECTED_PIPELINES); + verifyPutPipelines(unsuccessfulGetPipelines); + verifyWatcherCheck(); + verifyGetWatches(0); + verifyPutWatches(0); + verifyDeleteWatches(EXPECTED_WATCHES); + verifyNoMoreInteractions(client); + } + public void testSuccessfulChecksOnElectedMasterNode() { final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES); final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates; diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java index acc730f0e3dd3..af5c86ab0b085 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java @@ -32,7 +32,7 @@ public static void setupThreadPool() { } @AfterClass - public static void cleanUpStatic() throws Exception { + public static void cleanUpStatic() { if (THREADPOOL != null) { terminate(THREADPOOL); } @@ -57,6 +57,15 @@ protected Settings nodeSettings(int nodeOrdinal) { .build(); } + /** + * Create a new {@link LocalExporter} with the default exporter settings and name. + * + * @return Never {@code null}. + */ + protected LocalExporter createLocalExporter() { + return createLocalExporter(exporterName, null); + } + /** * Create a new {@link LocalExporter}. Expected usage: * 

@@ -68,12 +77,11 @@ protected Settings nodeSettings(int nodeOrdinal) {
      *
      * @return Never {@code null}.
      */
-    protected LocalExporter createLocalExporter() {
-        final Settings settings = localExporterSettings();
+    protected LocalExporter createLocalExporter(String exporterName, Settings exporterSettings) {
         final XPackLicenseState licenseState = TestUtils.newTestLicenseState();
-        final Exporter.Config config = new Exporter.Config(exporterName, "local", settings, clusterService(), licenseState);
+        final Exporter.Config config = new Exporter.Config(exporterName, "local", exporterSettings, clusterService(), licenseState);
         final CleanerService cleanerService =
-                new CleanerService(settings, clusterService().getClusterSettings(), THREADPOOL, licenseState);
+            new CleanerService(exporterSettings, clusterService().getClusterSettings(), THREADPOOL, licenseState);
 
         return new LocalExporter(config, client(), cleanerService);
     }
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
index 3c95af99964f1..aa1573b5e4011 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterResourceIntegTests.java
@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.monitoring.exporter.local;
 
 import org.elasticsearch.Version;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -25,8 +26,10 @@
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -75,6 +78,25 @@ public void testCreateWhenResourcesShouldNotBeReplaced() throws Exception {
         assertPipelinesNotUpdated();
     }
 
+    public void testRemoveWhenResourcesShouldBeRemoved() throws Exception {
+        putResources(newEnoughVersion());
+
+        assertResourcesExist();
+        waitNoPendingTasksOnAll();
+
+        Settings exporterSettings = Settings.builder().put(localExporterSettings())
+            .put("xpack.monitoring.migration.decommission_alerts", true).build();
+
+        createResources("decommission_local", exporterSettings);
+        waitNoPendingTasksOnAll();
+        assertBusy(() -> {
+            assertTemplatesExist();
+            assertPipelinesExist();
+            assertNoWatchesExist();
+        });
+
+    }
+
     @Override
     protected Settings localExporterSettings() {
         // Override the settings for local exporters created in this test, make sure watcher is enabled so we can test
@@ -86,11 +108,15 @@ protected Settings localExporterSettings() {
     }
 
     private void createResources() throws Exception {
+        createResources(exporterName, localExporterSettings());
+    }
+
+    private void createResources(String exporterName, Settings exporterSettings) throws Exception {
         // wait until the cluster is ready (this is done at the "Exporters" level)
         // this is not a busy assertion because it's checked earlier
         assertThat(clusterService().state().version(), not(ClusterState.UNKNOWN_VERSION));
 
-        try (LocalExporter exporter = createLocalExporter()) {
+        try (LocalExporter exporter = createLocalExporter(exporterName, exporterSettings)) {
             assertBusy(() -> assertThat(exporter.isExporterReady(), is(true)));
         }
     }
@@ -283,6 +309,26 @@ private void assertWatchesExist() {
         }
     }
 
+    private void assertNoWatchesExist() {
+        // Check if watches index exists
+        if (client().admin().indices().prepareGetIndex().addIndices(".watches").get().getIndices().length == 0) {
+            fail("Expected [.watches] index with cluster alerts present, but no [.watches] index was found");
+        }
+
+        String clusterUUID = clusterService().state().getMetadata().clusterUUID();
+        SearchSourceBuilder searchSource = SearchSourceBuilder.searchSource()
+            .query(QueryBuilders.matchQuery("metadata.xpack.cluster_uuid", clusterUUID));
+        SearchResponse searchResponse = client().prepareSearch(".watches").setSource(searchSource).get();
+        if (searchResponse.getHits().getTotalHits().value > 0) {
+            List invalidWatches = new ArrayList<>();
+            for (SearchHit hit : searchResponse.getHits().getHits()) {
+                invalidWatches.add(ObjectPath.eval("metadata.xpack.watch", hit.getSourceAsMap()));
+            }
+            fail("Found [" + searchResponse.getHits().getTotalHits().value + "] invalid watches when none were expected: "
+                + invalidWatches);
+        }
+    }
+
     private void assertResourcesExist() throws Exception {
         createResources();
 

From f4beb02959071341b4387174c6b270a41f1f6863 Mon Sep 17 00:00:00 2001
From: James Baiera 
Date: Fri, 18 Sep 2020 17:02:52 -0400
Subject: [PATCH 4/6] Appease the precommit beast

---
 .../xpack/monitoring/exporter/http/HttpExporter.java          | 1 -
 .../xpack/monitoring/exporter/local/LocalExporter.java        | 3 ++-
 .../monitoring/exporter/http/HttpExporterResourceTests.java   | 3 ++-
 .../xpack/monitoring/test/MockClusterAlertScriptEngine.java   | 4 ++--
 .../xpack/monitoring/test/MonitoringIntegTestCase.java        | 1 -
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java
index 875c3acb482be..37a5199d9eeda 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java
@@ -48,7 +48,6 @@
 import org.elasticsearch.xpack.monitoring.exporter.ExportBulk;
 import org.elasticsearch.xpack.monitoring.exporter.Exporter;
 
-import javax.management.monitor.Monitor;
 import javax.net.ssl.SSLContext;
 import java.util.ArrayList;
 import java.util.Arrays;
diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
index 2e167164534e2..fc01e7441f5f9 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
@@ -467,7 +467,8 @@ private void getClusterAlertsInstallationAsyncActions(final boolean indexExists,
 
         for (final String watchId : ClusterAlertsUtil.WATCH_IDS) {
             final String uniqueWatchId = ClusterAlertsUtil.createUniqueWatchId(clusterService, watchId);
-            final boolean addWatch = canAddWatches && clusterAlertBlacklist.contains(watchId) == false && decommissionClusterAlerts == false;
+            final boolean addWatch = canAddWatches && clusterAlertBlacklist.contains(watchId) == false &&
+                decommissionClusterAlerts == false;
 
             // we aren't sure if no watches exist yet, so add them
             if (indexExists) {
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
index d93d5a7a1df5c..ddab0f01a05c0 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
@@ -506,7 +506,8 @@ public void testWatchRemovalOnDecomissionAlerts() {
         // a number of watches are mocked as present
         final int existingWatches = randomIntBetween(0, EXPECTED_WATCHES);
 
-        // For completeness's sake. GET/PUT watches wont be called by the resources. Instead it tries to DELETE the watches ignoring them not existing.
+        // For completeness's sake. GET/PUT watches wont be called by the resources.
+        // Instead it tries to DELETE the watches ignoring them not existing.
         whenGetWatches(existingWatches, EXPECTED_WATCHES - existingWatches);
         whenPerformRequestAsyncWith(client, new RequestMatcher(is("PUT"), startsWith("/_watcher/watch/")), exception);
         whenPerformRequestAsyncWith(client, new RequestMatcher(is("DELETE"), startsWith("/_watcher/watch/")),
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockClusterAlertScriptEngine.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockClusterAlertScriptEngine.java
index 9b5ad1a15dc3e..77860f20d5e78 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockClusterAlertScriptEngine.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MockClusterAlertScriptEngine.java
@@ -62,7 +62,7 @@ static class MockWatcherConditionScript implements WatcherConditionScript.Factor
 
         private final MockDeterministicScript script;
 
-        public MockWatcherConditionScript(MockDeterministicScript script) {
+        MockWatcherConditionScript(MockDeterministicScript script) {
             this.script = script;
         }
 
@@ -84,7 +84,7 @@ static class MockWatcherTransformScript implements WatcherTransformScript.Factor
 
         private final MockDeterministicScript script;
 
-        public MockWatcherTransformScript(MockDeterministicScript script) {
+        MockWatcherTransformScript(MockDeterministicScript script) {
             this.script = script;
         }
 
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
index fb81519982559..a3303cbeaae37 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/test/MonitoringIntegTestCase.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.test.store.MockFSIndexStore;
 import org.elasticsearch.test.transport.MockTransportService;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
-import org.elasticsearch.xpack.core.monitoring.test.MockPainlessScriptEngine;
 import org.elasticsearch.xpack.monitoring.LocalStateMonitoring;
 import org.elasticsearch.xpack.monitoring.MonitoringService;
 import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;

From 2faf8e2e825e25db36dde44545b75c53a3de3934 Mon Sep 17 00:00:00 2001
From: James Baiera 
Date: Thu, 24 Sep 2020 15:54:43 -0400
Subject: [PATCH 5/6] Update
 x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java

Co-authored-by: Przemko Robakowski 
---
 .../xpack/monitoring/exporter/local/LocalExporter.java          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
index fc01e7441f5f9..df01140ad7ddd 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporter.java
@@ -163,7 +163,7 @@ boolean isExporterReady() {
         // forces the setup to occur if it hasn't already
         final boolean running = resolveBulk(clusterService.state(), false) != null;
         // Report on watcher readiness
-        boolean alertsProcessed = !canUseWatcher() || watcherSetup.get();
+        boolean alertsProcessed = canUseWatcher() == false || watcherSetup.get();
 
         return running && installingSomething.get() == false && alertsProcessed;
     }

From 3735e1a3cb0e019d6991bf555952ff7a592e132c Mon Sep 17 00:00:00 2001
From: James Baiera 
Date: Wed, 28 Oct 2020 14:32:50 -0400
Subject: [PATCH 6/6] addressed feedback

---
 .../java/org/elasticsearch/xpack/monitoring/Monitoring.java     | 1 +
 .../monitoring/exporter/http/HttpExporterResourceTests.java     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
index 9595427c59e54..23b8ccd004c10 100644
--- a/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
+++ b/x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java
@@ -149,6 +149,7 @@ public List> getSettings() {
         List> settings = new ArrayList<>();
         settings.add(MonitoringField.HISTORY_DURATION);
         settings.add(CLEAN_WATCHER_HISTORY);
+        settings.add(MIGRATION_DECOMMISSION_ALERTS);
         settings.add(MonitoringService.ENABLED);
         settings.add(MonitoringService.ELASTICSEARCH_COLLECTION_ENABLED);
         settings.add(MonitoringService.INTERVAL);
diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
index ddab0f01a05c0..c64bd2af452b8 100644
--- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
+++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterResourceTests.java
@@ -488,7 +488,7 @@ public void testWatchPublishBlocksAfterSuccessfulWatcherCheck() {
         verifyNoMoreInteractions(client);
     }
 
-    public void testWatchRemovalOnDecomissionAlerts() {
+    public void testDeployClusterAlerts() {
         final int successfulGetTemplates = randomIntBetween(0, EXPECTED_TEMPLATES);
         final int unsuccessfulGetTemplates = EXPECTED_TEMPLATES - successfulGetTemplates;
         final int successfulGetPipelines = randomIntBetween(0, EXPECTED_PIPELINES);