Skip to content

Commit abda989

Browse files
jbaierajbaiera
authored
Add action to decommission legacy monitoring cluster alerts (#64373)
Adds an action that will proactively remove any watches that monitoring has configured. The action toggles on a new setting that informs the cluster to tear down any previously created cluster alerts, and after that is accepted, the action immediately attempts a best-effort refresh of cluster alert resources in order to force their removal in case collection is disabled or delayed. Since resources are controlled lazily by the existing monitoring exporters, extra care was taken to ensure that any in-flight resource management operations do not race against any resource actions taken by the migration action. Resource installation code was updated with callbacks to report any errors instead of just logging them.
1 parent ea2145a commit abda989

File tree

38 files changed

+2040
-228
lines changed

38 files changed

+2040
-228
lines changed

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringMigrateAlertsAction.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.core.monitoring.action;
7+
8+
import org.elasticsearch.action.ActionType;
9+
10+
public class MonitoringMigrateAlertsAction extends ActionType<MonitoringMigrateAlertsResponse> {
11+
12+
public static final MonitoringMigrateAlertsAction INSTANCE = new MonitoringMigrateAlertsAction();
13+
public static final String NAME = "cluster:admin/xpack/monitoring/migrate/alerts";
14+
15+
public MonitoringMigrateAlertsAction() {
16+
super(NAME, MonitoringMigrateAlertsResponse::new);
17+
}
18+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringMigrateAlertsRequest.java

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.core.monitoring.action;
8+
9+
import org.elasticsearch.action.ActionRequestValidationException;
10+
import org.elasticsearch.action.support.master.MasterNodeRequest;
11+
import org.elasticsearch.common.io.stream.StreamInput;
12+
13+
import java.io.IOException;
14+
15+
public class MonitoringMigrateAlertsRequest extends MasterNodeRequest<MonitoringMigrateAlertsRequest> {
16+
17+
public MonitoringMigrateAlertsRequest() {}
18+
19+
public MonitoringMigrateAlertsRequest(StreamInput in) throws IOException {
20+
super(in);
21+
}
22+
23+
@Override
24+
public ActionRequestValidationException validate() {
25+
return null;
26+
}
27+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringMigrateAlertsResponse.java

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.core.monitoring.action;
8+
9+
import org.elasticsearch.ElasticsearchException;
10+
import org.elasticsearch.action.ActionResponse;
11+
import org.elasticsearch.common.Nullable;
12+
import org.elasticsearch.common.io.stream.StreamInput;
13+
import org.elasticsearch.common.io.stream.StreamOutput;
14+
import org.elasticsearch.common.io.stream.Writeable;
15+
import org.elasticsearch.common.xcontent.ToXContent;
16+
import org.elasticsearch.common.xcontent.ToXContentObject;
17+
import org.elasticsearch.common.xcontent.XContentBuilder;
18+
19+
import java.io.IOException;
20+
import java.util.List;
21+
import java.util.Objects;
22+
23+
public class MonitoringMigrateAlertsResponse extends ActionResponse implements ToXContentObject {
24+
25+
private final List<ExporterMigrationResult> exporters;
26+
27+
public MonitoringMigrateAlertsResponse(List<ExporterMigrationResult> exporters) {
28+
this.exporters = exporters;
29+
}
30+
31+
public MonitoringMigrateAlertsResponse(StreamInput in) throws IOException {
32+
super(in);
33+
this.exporters = in.readList(ExporterMigrationResult::new);
34+
}
35+
36+
@Override
37+
public void writeTo(StreamOutput out) throws IOException {
38+
out.writeList(exporters);
39+
}
40+
41+
@Override
42+
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
43+
return builder.startObject()
44+
.array("exporters", exporters)
45+
.endObject();
46+
}
47+
48+
public List<ExporterMigrationResult> getExporters() {
49+
return exporters;
50+
}
51+
52+
@Override
53+
public boolean equals(Object o) {
54+
if (this == o) return true;
55+
if (o == null || getClass() != o.getClass()) return false;
56+
MonitoringMigrateAlertsResponse response = (MonitoringMigrateAlertsResponse) o;
57+
return Objects.equals(exporters, response.exporters);
58+
}
59+
60+
@Override
61+
public int hashCode() {
62+
return Objects.hash(exporters);
63+
}
64+
65+
@Override
66+
public String toString() {
67+
return "MonitoringMigrateAlertsResponse{" +
68+
"exporters=" + exporters +
69+
'}';
70+
}
71+
72+
public static class ExporterMigrationResult implements Writeable, ToXContentObject {
73+
74+
private final String name;
75+
private final String type;
76+
private final boolean migrationComplete;
77+
private final Exception reason;
78+
79+
public ExporterMigrationResult(String name, String type, boolean migrationComplete, Exception reason) {
80+
this.name = name;
81+
this.type = type;
82+
this.migrationComplete = migrationComplete;
83+
this.reason = reason;
84+
}
85+
86+
public ExporterMigrationResult(StreamInput in) throws IOException {
87+
this.name = in.readString();
88+
this.type = in.readString();
89+
this.migrationComplete = in.readBoolean();
90+
this.reason = in.readException();
91+
}
92+
93+
@Override
94+
public void writeTo(StreamOutput out) throws IOException {
95+
out.writeString(name);
96+
out.writeString(type);
97+
out.writeBoolean(migrationComplete);
98+
out.writeException(reason);
99+
}
100+
101+
@Override
102+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
103+
builder.startObject();
104+
{
105+
builder.field("name", name);
106+
builder.field("type", type);
107+
builder.field("migration_complete", migrationComplete);
108+
if (reason != null) {
109+
builder.startObject("reason");
110+
ElasticsearchException.generateThrowableXContent(builder, params, reason);
111+
builder.endObject();
112+
}
113+
}
114+
return builder.endObject();
115+
}
116+
117+
public String getName() {
118+
return name;
119+
}
120+
121+
public String getType() {
122+
return type;
123+
}
124+
125+
public boolean isMigrationComplete() {
126+
return migrationComplete;
127+
}
128+
129+
@Nullable
130+
public Exception getReason() {
131+
return reason;
132+
}
133+
134+
@Override
135+
public boolean equals(Object o) {
136+
if (this == o) return true;
137+
if (o == null || getClass() != o.getClass()) return false;
138+
ExporterMigrationResult that = (ExporterMigrationResult) o;
139+
return migrationComplete == that.migrationComplete &&
140+
Objects.equals(name, that.name) &&
141+
Objects.equals(type, that.type);
142+
}
143+
144+
@Override
145+
public int hashCode() {
146+
return Objects.hash(name, type, migrationComplete);
147+
}
148+
149+
@Override
150+
public String toString() {
151+
return "ExporterMigrationResult{" +
152+
"name='" + name + '\'' +
153+
", type='" + type + '\'' +
154+
", migrationComplete=" + migrationComplete +
155+
", reason=" + reason +
156+
'}';
157+
}
158+
}
159+
}

‎x-pack/plugin/monitoring/src/internalClusterTest/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporterIT.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@
5252
import org.elasticsearch.xpack.monitoring.exporter.ClusterAlertsUtil;
5353
import org.elasticsearch.xpack.monitoring.exporter.ExportBulk;
5454
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
55+
import org.elasticsearch.xpack.monitoring.exporter.MonitoringMigrationCoordinator;
5556
import org.elasticsearch.xpack.monitoring.test.MonitoringIntegTestCase;
5657
import org.junit.After;
5758
import org.junit.Before;
@@ -99,6 +100,7 @@ public class HttpExporterIT extends MonitoringIntegTestCase {
99100
private final boolean watcherAlreadyExists = randomBoolean();
100101
private final Environment environment = TestEnvironment.newEnvironment(Settings.builder().put("path.home", createTempDir()).build());
101102
private final String userName = "elasticuser";
103+
private final MonitoringMigrationCoordinator coordinator = new MonitoringMigrationCoordinator();
102104

103105
private MockWebServer webServer;
104106

@@ -651,7 +653,7 @@ private HttpExporter createHttpExporter(final Settings settings) {
651653
new Exporter.Config("_http", "http", settings, clusterService(), TestUtils.newTestLicenseState());
652654

653655
final Environment env = TestEnvironment.newEnvironment(buildEnvSettings(settings));
654-
return new HttpExporter(config, new SSLService(env), new ThreadContext(settings));
656+
return new HttpExporter(config, new SSLService(env), new ThreadContext(settings), coordinator);
655657
}
656658

657659
private void export(final Settings settings, final Collection<MonitoringDoc> docs) throws Exception {

‎x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monitoring.java

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,10 @@
3636
import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
3737
import org.elasticsearch.xpack.core.monitoring.MonitoringField;
3838
import org.elasticsearch.xpack.core.monitoring.action.MonitoringBulkAction;
39+
import org.elasticsearch.xpack.core.monitoring.action.MonitoringMigrateAlertsAction;
3940
import org.elasticsearch.xpack.core.ssl.SSLService;
4041
import org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction;
42+
import org.elasticsearch.xpack.monitoring.action.TransportMonitoringMigrateAlertsAction;
4143
import org.elasticsearch.xpack.monitoring.cleaner.CleanerService;
4244
import org.elasticsearch.xpack.monitoring.collector.Collector;
4345
import org.elasticsearch.xpack.monitoring.collector.ccr.StatsCollector;
@@ -50,9 +52,11 @@
5052
import org.elasticsearch.xpack.monitoring.collector.shards.ShardsCollector;
5153
import org.elasticsearch.xpack.monitoring.exporter.Exporter;
5254
import org.elasticsearch.xpack.monitoring.exporter.Exporters;
55+
import org.elasticsearch.xpack.monitoring.exporter.MonitoringMigrationCoordinator;
5356
import org.elasticsearch.xpack.monitoring.exporter.http.HttpExporter;
5457
import org.elasticsearch.xpack.monitoring.exporter.local.LocalExporter;
5558
import org.elasticsearch.xpack.monitoring.rest.action.RestMonitoringBulkAction;
59+
import org.elasticsearch.xpack.monitoring.rest.action.RestMonitoringMigrateAlertsAction;
5660

5761
import java.util.ArrayList;
5862
import java.util.Arrays;
@@ -65,7 +69,6 @@
6569
import java.util.Set;
6670
import java.util.function.Supplier;
6771

68-
import static java.util.Collections.singletonList;
6972
import static org.elasticsearch.common.settings.Setting.boolSetting;
7073

7174
public class Monitoring extends Plugin implements ActionPlugin, ReloadablePlugin {
@@ -103,10 +106,12 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
103106
final ClusterSettings clusterSettings = clusterService.getClusterSettings();
104107
final CleanerService cleanerService = new CleanerService(settings, clusterSettings, threadPool, getLicenseState());
105108
final SSLService dynamicSSLService = getSslService().createDynamicSSLService();
109+
final MonitoringMigrationCoordinator migrationCoordinator = new MonitoringMigrationCoordinator();
106110

107111
Map<String, Exporter.Factory> exporterFactories = new HashMap<>();
108-
exporterFactories.put(HttpExporter.TYPE, config -> new HttpExporter(config, dynamicSSLService, threadPool.getThreadContext()));
109-
exporterFactories.put(LocalExporter.TYPE, config -> new LocalExporter(config, client, cleanerService));
112+
exporterFactories.put(HttpExporter.TYPE, config -> new HttpExporter(config, dynamicSSLService, threadPool.getThreadContext(),
113+
migrationCoordinator));
114+
exporterFactories.put(LocalExporter.TYPE, config -> new LocalExporter(config, client, migrationCoordinator, cleanerService));
110115
exporters = new Exporters(settings, exporterFactories, clusterService, getLicenseState(), threadPool.getThreadContext(),
111116
dynamicSSLService);
112117

@@ -124,7 +129,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
124129
final MonitoringService monitoringService = new MonitoringService(settings, clusterService, threadPool, collectors, exporters);
125130

126131
var usageServices = new MonitoringUsageServices(monitoringService, exporters);
127-
return Arrays.asList(monitoringService, exporters, cleanerService, usageServices);
132+
return Arrays.asList(monitoringService, exporters, migrationCoordinator, cleanerService, usageServices);
128133
}
129134

130135
@Override
@@ -133,6 +138,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
133138
var infoAction = new ActionHandler<>(XPackInfoFeatureAction.MONITORING, MonitoringInfoTransportAction.class);
134139
return Arrays.asList(
135140
new ActionHandler<>(MonitoringBulkAction.INSTANCE, TransportMonitoringBulkAction.class),
141+
new ActionHandler<>(MonitoringMigrateAlertsAction.INSTANCE, TransportMonitoringMigrateAlertsAction.class),
136142
usageAction,
137143
infoAction);
138144
}
@@ -141,15 +147,14 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
141147
public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
142148
IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
143149
Supplier<DiscoveryNodes> nodesInCluster) {
144-
return singletonList(new RestMonitoringBulkAction());
150+
return List.of(new RestMonitoringBulkAction(), new RestMonitoringMigrateAlertsAction());
145151
}
146152

147153
@Override
148154
public List<Setting<?>> getSettings() {
149155
List<Setting<?>> settings = new ArrayList<>();
150156
settings.add(MonitoringField.HISTORY_DURATION);
151157
settings.add(CLEAN_WATCHER_HISTORY);
152-
settings.add(MIGRATION_DECOMMISSION_ALERTS);
153158
settings.add(MonitoringService.ENABLED);
154159
settings.add(MonitoringService.ELASTICSEARCH_COLLECTION_ENABLED);
155160
settings.add(MonitoringService.INTERVAL);
@@ -163,6 +168,7 @@ public List<Setting<?>> getSettings() {
163168
settings.add(NodeStatsCollector.NODE_STATS_TIMEOUT);
164169
settings.add(EnrichStatsCollector.STATS_TIMEOUT);
165170
settings.addAll(Exporters.getSettings());
171+
settings.add(Monitoring.MIGRATION_DECOMMISSION_ALERTS);
166172
return Collections.unmodifiableList(settings);
167173
}
168174

0 commit comments

Comments
 (0)