From ca6a8b61e04f33a8bf57aa87bc580ffc66cc4e04 Mon Sep 17 00:00:00 2001 From: Gordon Brown Date: Tue, 11 Feb 2020 12:53:40 -0700 Subject: [PATCH] Convert ILM and SLM histories into hidden indices (#51456) Modifies SLM's and ILM's history indices to be hidden indices for added protection against accidental querying and deletion, and improves IndexTemplateRegistry to handle upgrading index templates. Also modifies the REST test cleanup to delete hidden indices. --- .../metadata/MetaDataCreateIndexService.java | 1 - .../MetaDataCreateIndexServiceTests.java | 1 - .../test/rest/ESRestTestCase.java | 7 +- .../SnapshotLifecycleTemplateRegistry.java | 3 +- .../core/template/IndexTemplateConfig.java | 19 ++- .../core/template/IndexTemplateRegistry.java | 20 ++- .../xpack/core/template/TemplateUtils.java | 5 +- .../WatcherIndexTemplateRegistryField.java | 2 +- .../core/src/main/resources/ilm-history.json | 4 +- .../core/src/main/resources/slm-history.json | 6 +- .../src/main/resources/triggered-watches.json | 3 +- .../main/resources/watch-history-no-ilm.json | 3 +- .../src/main/resources/watch-history.json | 3 +- .../core/src/main/resources/watches.json | 3 +- ...napshotLifecycleTemplateRegistryTests.java | 140 +++++++++++++----- .../history/ILMHistoryTemplateRegistry.java | 3 +- .../watcher/history/HistoryStoreTests.java | 2 +- .../WatcherIndexTemplateRegistryTests.java | 108 +++++++++----- 18 files changed, 234 insertions(+), 99 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java index bc70ca0b9c5f2..9f972ec5b8076 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java @@ -120,7 +120,6 @@ public class MetaDataCreateIndexService { * These index patterns will be converted to hidden indices, at which point they should be removed from this list. */ private static final CharacterRunAutomaton DOT_INDICES_EXCLUSIONS = new CharacterRunAutomaton(Regex.simpleMatchToAutomaton( - ".slm-history-*", ".watch-history-*", ".ml-anomalies-*", ".ml-notifications-*", diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java index 7b014a21ef828..8a37659147ee1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexServiceTests.java @@ -641,7 +641,6 @@ public void testValidateDotIndex() { public void testIndexNameExclusionsList() { // this test case should be removed when DOT_INDICES_EXCLUSIONS is empty List excludedNames = Arrays.asList( - ".slm-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".watch-history-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".ml-anomalies-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), ".ml-notifications-" + randomAlphaOfLength(5).toLowerCase(Locale.ROOT), diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index aae90b7c37db0..6d999a5b2cf0a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -599,8 +599,11 @@ private void wipeCluster() throws Exception { } protected static void wipeAllIndices() throws IOException { + boolean includeHidden = minimumNodeVersion().onOrAfter(Version.V_7_7_0); try { - final Response response = adminClient().performRequest(new Request("DELETE", "*")); + final Request deleteReq = new Request("DELETE", "*"); + deleteReq.addParameter("expand_wildcards", "open,closed" + (includeHidden ? ",hidden" : "")); + final Response response = adminClient().performRequest(deleteReq); try (InputStream is = response.getEntity().getContent()) { assertTrue((boolean) XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true).get("acknowledged")); } @@ -1235,7 +1238,7 @@ protected static Version minimumNodeVersion() throws IOException { final Request request = new Request("GET", "_nodes"); request.addParameter("filter_path", "nodes.*.version"); - final Response response = client().performRequest(request); + final Response response = adminClient().performRequest(request); final Map nodes = ObjectPath.createFromResponse(response).evaluate("nodes"); Version minVersion = null; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index 659ef74b3fd2a..ef768cd992c37 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -35,7 +35,8 @@ public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry { // history (please add a comment why you increased the version here) // version 1: initial - public static final String INDEX_TEMPLATE_VERSION = "1"; + // version 2: converted to hidden index + public static final int INDEX_TEMPLATE_VERSION = 2; public static final String SLM_TEMPLATE_VERSION_VARIABLE = "xpack.slm.template.version"; public static final String SLM_TEMPLATE_NAME = ".slm-history"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java index 5eb219f11a7a0..4565f7a93b149 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateConfig.java @@ -16,7 +16,7 @@ public class IndexTemplateConfig { private final String templateName; private final String fileName; - private final String version; + private final int version; private final String versionProperty; /** @@ -27,14 +27,17 @@ public class IndexTemplateConfig { * {@code {"myTemplateVersion": "${my.version.property}"}} * With {@code version = "42"; versionProperty = "my.version.property"} will result in {@code {"myTemplateVersion": "42"}}. * - * @param templateName The name that will be used for the index template. Literal, include the version in this string if + * Note that this code does not automatically insert the {@code version} index template property - include that in the JSON file + * defining the template, preferably using the version variable provided to this constructor. + * + * @param templateName The name that will be used for the index template. Literal, include the version in this string if * it should be used. * @param fileName The filename the template should be loaded from. Literal, should include leading {@literal /} and * extension if necessary. * @param version The version of the template. Substituted for {@code versionProperty} as described above. * @param versionProperty The property that will be replaced with the {@code version} string as described above. */ - public IndexTemplateConfig(String templateName, String fileName, String version, String versionProperty) { + public IndexTemplateConfig(String templateName, String fileName, int version, String versionProperty) { this.templateName = templateName; this.fileName = fileName; this.version = version; @@ -49,14 +52,20 @@ public String getTemplateName() { return templateName; } + public int getVersion() { + return version; + } + /** * Loads the template from disk as a UTF-8 byte array. * @return The template as a UTF-8 byte array. */ public byte[] loadBytes() { - String template = TemplateUtils.loadTemplate(fileName, version, - Pattern.quote("${" + versionProperty + "}")); + final String versionPattern = Pattern.quote("${" + versionProperty + "}"); + String template = TemplateUtils.loadTemplate(fileName, Integer.toString(version), versionPattern); assert template != null && template.length() > 0; + assert Pattern.compile("\"version\"\\s*:\\s*" + version).matcher(template).find() + : "index template must have a version property set to the given version property"; return template.getBytes(StandardCharsets.UTF_8); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index 870887e9b0818..f5d95204a7995 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -16,6 +16,7 @@ import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateListener; +import org.elasticsearch.cluster.metadata.IndexTemplateMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -31,6 +32,7 @@ import org.elasticsearch.xpack.core.ilm.action.PutLifecycleAction; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -130,16 +132,24 @@ public void clusterChanged(ClusterChangedEvent event) { private void addTemplatesIfMissing(ClusterState state) { final List indexTemplates = getTemplateConfigs(); - for (IndexTemplateConfig template : indexTemplates) { - final String templateName = template.getTemplateName(); + for (IndexTemplateConfig newTemplate : indexTemplates) { + final String templateName = newTemplate.getTemplateName(); final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false)); if (creationCheck.compareAndSet(false, true)) { - if (!state.metaData().getTemplates().containsKey(templateName)) { + IndexTemplateMetaData currentTemplate = state.metaData().getTemplates().get(templateName); + if (Objects.isNull(currentTemplate)) { logger.debug("adding index template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); - putTemplate(template, creationCheck); + putTemplate(newTemplate, creationCheck); + } else if (Objects.isNull(currentTemplate.getVersion()) || newTemplate.getVersion() > currentTemplate.getVersion()) { + // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can + // safely assume it's an old version of the template. + logger.info("upgrading index template [{}] for [{}] from version [{}] to version [{}]", + templateName, getOrigin(), currentTemplate.getVersion(), newTemplate.getVersion()); + putTemplate(newTemplate, creationCheck); } else { creationCheck.set(false); - logger.trace("not adding index template [{}] for [{}], because it already exists", templateName, getOrigin()); + logger.trace("not adding index template [{}] for [{}], because it already exists at version [{}]", + templateName, getOrigin(), currentTemplate.getVersion()); } } else { logger.trace("skipping the creation of index template [{}] for [{}], because its creation is in progress", diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java index 6b25f7855f107..3c7e9cdfb9095 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/TemplateUtils.java @@ -59,9 +59,10 @@ public static void loadTemplateIntoMap(String resource, Map { - if (action instanceof PutIndexTemplateAction) { - calledTimes.incrementAndGet(); - assertThat(action, instanceOf(PutIndexTemplateAction.class)); - assertThat(request, instanceOf(PutIndexTemplateRequest.class)); - final PutIndexTemplateRequest putRequest = (PutIndexTemplateRequest) request; - assertThat(putRequest.name(), equalTo(SLM_TEMPLATE_NAME)); - assertThat(putRequest.settings().get("index.lifecycle.name"), equalTo(SLM_POLICY_NAME)); - assertNotNull(listener); - return new TestPutIndexTemplateResponse(true); - } else if (action instanceof PutLifecycleAction) { - // Ignore this, it's verified in another test - return new PutLifecycleAction.Response(true); - } else { - fail("client called with unexpected request:" + request.toString()); - return null; - } - }); + client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); registry.clusterChanged(event); assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); @@ -145,7 +129,7 @@ public void testThatNonExistingTemplatesAreAddedImmediately() throws Exception { // will not be issued anymore, leaving calledTimes to 0 assertBusy(() -> { // now delete one template from the cluster state and lets retry - ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyList(), nodes); + ClusterChangedEvent newEvent = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(newEvent); assertThat(calledTimes.get(), greaterThan(1)); }); @@ -174,7 +158,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception { } }); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(event); assertBusy(() -> assertThat(calledTimes.get(), equalTo(1))); } @@ -203,7 +187,7 @@ public void testPolicyAlreadyExists() { return null; }); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); registry.clusterChanged(event); } @@ -235,11 +219,61 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException { .createParser(xContentRegistry, LoggingDeprecationHandler.THROW_UNSUPPORTED_OPERATION, policyStr)) { LifecyclePolicy different = LifecyclePolicy.parse(parser, policy.getName()); policyMap.put(policy.getName(), different); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); registry.clusterChanged(event); } } + public void testThatVersionedOldTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION - 1), + nodes); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + } + + public void testThatUnversionedOldTemplatesAreUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap(SLM_TEMPLATE_NAME, null), nodes); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> verifyTemplateInstalled(calledTimes, action, request, listener)); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(registry.getTemplateConfigs().size()))); + } + + + public void testSameOrHigherVersionTemplateNotUpgraded() throws Exception { + DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterChangedEvent sameVersionEvent = createClusterChangedEvent( + Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION), nodes); + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutIndexTemplateAction) { + fail("template should not have been re-installed"); + return null; + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return new PutLifecycleAction.Response(true); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + }); + registry.clusterChanged(sameVersionEvent); + + ClusterChangedEvent higherVersionEvent = createClusterChangedEvent( + Collections.singletonMap(SLM_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION + randomIntBetween(1, 1000)), nodes); + registry.clusterChanged(higherVersionEvent); + } + public void testThatMissingMasterNodeDoesNothing() { DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); @@ -249,20 +283,25 @@ public void testThatMissingMasterNodeDoesNothing() { return null; }); - ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(SLM_TEMPLATE_NAME), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap(SLM_TEMPLATE_NAME, null), nodes); registry.clusterChanged(event); } public void testValidate() { - assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyList(), Collections.emptyMap(), null))); - assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.singletonList(SLM_TEMPLATE_NAME), - Collections.emptyMap(), null))); + assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), null))); + assertFalse(registry.validate(createClusterState(Settings.EMPTY, + Collections.singletonMap(SLM_TEMPLATE_NAME, null), + Collections.emptyMap(), + null))); Map policyMap = new HashMap<>(); policyMap.put(SLM_POLICY_NAME, new LifecyclePolicy(SLM_POLICY_NAME, new HashMap<>())); - assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyList(), policyMap, null))); + assertFalse(registry.validate(createClusterState(Settings.EMPTY, Collections.emptyMap(), policyMap, null))); - assertTrue(registry.validate(createClusterState(Settings.EMPTY, Collections.singletonList(SLM_TEMPLATE_NAME), policyMap, null))); + assertTrue(registry.validate(createClusterState(Settings.EMPTY, + Collections.singletonMap(SLM_TEMPLATE_NAME, null), + policyMap, + null))); } // ------------- @@ -299,14 +338,35 @@ public VerifyingClient setVerifier(TriFunction, ActionRequest, Act } } - private ClusterChangedEvent createClusterChangedEvent(List existingTemplateNames, DiscoveryNodes nodes) { - return createClusterChangedEvent(existingTemplateNames, Collections.emptyMap(), nodes); + private ActionResponse verifyTemplateInstalled( + AtomicInteger calledTimes, ActionType action, ActionRequest request, ActionListener listener) { + if (action instanceof PutIndexTemplateAction) { + calledTimes.incrementAndGet(); + assertThat(action, instanceOf(PutIndexTemplateAction.class)); + assertThat(request, instanceOf(PutIndexTemplateRequest.class)); + final PutIndexTemplateRequest putRequest = (PutIndexTemplateRequest) request; + assertThat(putRequest.name(), equalTo(SLM_TEMPLATE_NAME)); + assertThat(putRequest.settings().get("index.lifecycle.name"), equalTo(SLM_POLICY_NAME)); + assertThat(putRequest.version(), equalTo(INDEX_TEMPLATE_VERSION)); + assertNotNull(listener); + return new TestPutIndexTemplateResponse(true); + } else if (action instanceof PutLifecycleAction) { + // Ignore this, it's verified in another test + return new PutLifecycleAction.Response(true); + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + } + + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplates, DiscoveryNodes nodes) { + return createClusterChangedEvent(existingTemplates, Collections.emptyMap(), nodes); } - private ClusterChangedEvent createClusterChangedEvent(List existingTemplateNames, + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplates, Map existingPolicies, DiscoveryNodes nodes) { - ClusterState cs = createClusterState(Settings.EMPTY, existingTemplateNames, existingPolicies, nodes); + ClusterState cs = createClusterState(Settings.EMPTY, existingTemplates, existingPolicies, nodes); ClusterChangedEvent realEvent = new ClusterChangedEvent("created-from-test", cs, ClusterState.builder(new ClusterName("test")).build()); ClusterChangedEvent event = spy(realEvent); @@ -315,13 +375,15 @@ private ClusterChangedEvent createClusterChangedEvent(List existingTempl return event; } - private ClusterState createClusterState(Settings nodeSettings, - List existingTemplateNames, - Map existingPolicies, - DiscoveryNodes nodes) { + private ClusterState createClusterState(Settings nodeSettings, Map existingTemplates, + Map existingPolicies, DiscoveryNodes nodes) { ImmutableOpenMap.Builder indexTemplates = ImmutableOpenMap.builder(); - for (String name : existingTemplateNames) { - indexTemplates.put(name, mock(IndexTemplateMetaData.class)); + for (Map.Entry template : existingTemplates.entrySet()) { + final IndexTemplateMetaData mockTemplate = mock(IndexTemplateMetaData.class); + when(mockTemplate.version()).thenReturn(template.getValue()); + when(mockTemplate.getVersion()).thenReturn(template.getValue()); + + indexTemplates.put(template.getKey(), mockTemplate); } Map existingILMMeta = existingPolicies.entrySet().stream() diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index 21b2d16afdc83..149d4297cafeb 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -27,7 +27,8 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry { // history (please add a comment why you increased the version here) // version 1: initial - public static final String INDEX_TEMPLATE_VERSION = "1"; + // version 2: convert to hidden index + public static final int INDEX_TEMPLATE_VERSION = 2; public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version"; public static final String ILM_TEMPLATE_NAME = "ilm-history"; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java index 8a6e53ec02426..59aef01f180a1 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryStoreTests.java @@ -105,7 +105,7 @@ public void testPut() throws Exception { } public void testIndexNameGeneration() { - String indexTemplateVersion = INDEX_TEMPLATE_VERSION; + String indexTemplateVersion = Integer.toString(INDEX_TEMPLATE_VERSION); assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli((long) 0).atZone(ZoneOffset.UTC)), equalTo(".watcher-history-"+ indexTemplateVersion +"-1970.01.01")); assertThat(getHistoryIndexNameForTime(Instant.ofEpochMilli(100000000000L).atZone(ZoneOffset.UTC)), diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java index 85dc50c670ba8..f0be71155dc23 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java @@ -58,6 +58,7 @@ import static org.elasticsearch.mock.orig.Mockito.verify; import static org.elasticsearch.mock.orig.Mockito.when; +import static org.elasticsearch.xpack.core.watcher.support.WatcherIndexTemplateRegistryField.INDEX_TEMPLATE_VERSION; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -111,14 +112,16 @@ public void testThatNonExistingTemplatesAreAddedImmediately() { DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(event); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); verify(client.admin().indices(), times(3)).putTemplate(argumentCaptor.capture(), anyObject()); // now delete one template from the cluster state and lets retry - ClusterChangedEvent newEvent = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(newEvent); ArgumentCaptor captor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); verify(client.admin().indices(), times(4)).putTemplate(captor.capture(), anyObject()); @@ -136,14 +139,16 @@ public void testThatNonExistingTemplatesAreAddedEvenWithILMDisabled() { registry = new WatcherIndexTemplateRegistry(Settings.builder() .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false).build(), clusterService, threadPool, client, xContentRegistry); - ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyList(), Collections.emptyMap(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), nodes); registry.clusterChanged(event); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); verify(client.admin().indices(), times(3)).putTemplate(argumentCaptor.capture(), anyObject()); // now delete one template from the cluster state and lets retry - ClusterChangedEvent newEvent = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + ClusterChangedEvent newEvent = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(newEvent); ArgumentCaptor captor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); verify(client.admin().indices(), times(5)).putTemplate(captor.capture(), anyObject()); @@ -155,7 +160,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() { DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(event); verify(client, times(1)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } @@ -171,7 +176,7 @@ public void testPolicyAlreadyExists() { assertThat(policies, hasSize(1)); LifecyclePolicy policy = policies.get(0); policyMap.put(policy.getName(), policy); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); registry.clusterChanged(event); verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } @@ -183,7 +188,7 @@ public void testNoPolicyButILMDisabled() { registry = new WatcherIndexTemplateRegistry(Settings.builder() .put(XPackSettings.INDEX_LIFECYCLE_ENABLED.getKey(), false).build(), clusterService, threadPool, client, xContentRegistry); - ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyList(), Collections.emptyMap(), nodes); + ClusterChangedEvent event = createClusterChangedEvent(Settings.EMPTY, Collections.emptyMap(), Collections.emptyMap(), nodes); registry.clusterChanged(event); verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } @@ -203,20 +208,44 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException { .createParser(xContentRegistry, LoggingDeprecationHandler.THROW_UNSUPPORTED_OPERATION, policyStr)) { LifecyclePolicy different = LifecyclePolicy.parse(parser, policy.getName()); policyMap.put(policy.getName(), different); - ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyList(), policyMap, nodes); + ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), policyMap, nodes); registry.clusterChanged(event); verify(client, times(0)).execute(eq(PutLifecycleAction.INSTANCE), anyObject(), anyObject()); } } public void testThatTemplatesExist() { - assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history")), is(false)); - assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(".watch-history", ".triggered_watches", ".watches")), - is(false)); - assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, - ".triggered_watches", ".watches")), is(true)); - assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, - ".triggered_watches", ".watches", "whatever", "else")), is(true)); + { + Map existingTemplates = new HashMap<>(); + existingTemplates.put(".watch-history", INDEX_TEMPLATE_VERSION); + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(false)); + } + + { + Map existingTemplates = new HashMap<>(); + existingTemplates.put(".watch-history", INDEX_TEMPLATE_VERSION); + existingTemplates.put(".triggered_watches", INDEX_TEMPLATE_VERSION); + existingTemplates.put(".watches", INDEX_TEMPLATE_VERSION); + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(false)); + } + + { + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(".triggered_watches", INDEX_TEMPLATE_VERSION); + existingTemplates.put(".watches", INDEX_TEMPLATE_VERSION); + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(true)); + } + { + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(".triggered_watches", INDEX_TEMPLATE_VERSION); + existingTemplates.put(".watches", INDEX_TEMPLATE_VERSION); + existingTemplates.put("whatever", null); + existingTemplates.put("else", null); + + assertThat(WatcherIndexTemplateRegistry.validate(createClusterState(existingTemplates)), is(true)); + } } // if a node is newer than the master node, the template needs to be applied as well @@ -226,8 +255,11 @@ public void testThatTemplatesAreAppliedOnNewerNodes() { DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build(); - ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION); + existingTemplates.put(".watch-history-6", 6); + ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class); @@ -240,8 +272,11 @@ public void testThatTemplatesAreNotAppliedOnSameVersionNodes() { DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build(); - ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, null); + existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, null); + existingTemplates.put(".watch-history-6", null); + ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); verifyZeroInteractions(client); @@ -251,24 +286,30 @@ public void testThatMissingMasterNodeDoesNothing() { DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); - ClusterChangedEvent event = createClusterChangedEvent(Arrays.asList(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, - WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, ".watch-history-6"), nodes); + Map existingTemplates = new HashMap<>(); + existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, null); + existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, null); + existingTemplates.put(".watch-history-6", null); + ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes); registry.clusterChanged(event); verifyZeroInteractions(client); } - private ClusterChangedEvent createClusterChangedEvent(List existingTemplateNames, DiscoveryNodes nodes) { + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplateNames, DiscoveryNodes nodes) { return createClusterChangedEvent(existingTemplateNames, Collections.emptyMap(), nodes); } private ClusterState createClusterState(Settings nodeSettings, - List existingTemplateNames, + Map existingTemplates, Map existingPolicies, DiscoveryNodes nodes) { ImmutableOpenMap.Builder indexTemplates = ImmutableOpenMap.builder(); - for (String name : existingTemplateNames) { - indexTemplates.put(name, mock(IndexTemplateMetaData.class)); + for (Map.Entry template : existingTemplates.entrySet()) { + final IndexTemplateMetaData mockTemplate = mock(IndexTemplateMetaData.class); + when(mockTemplate.version()).thenReturn(template.getValue()); + when(mockTemplate.getVersion()).thenReturn(template.getValue()); + indexTemplates.put(template.getKey(), mockTemplate); } Map existingILMMeta = existingPolicies.entrySet().stream() @@ -286,17 +327,17 @@ private ClusterState createClusterState(Settings nodeSettings, .build(); } - private ClusterChangedEvent createClusterChangedEvent(List existingTemplateNames, + private ClusterChangedEvent createClusterChangedEvent(Map existingTemplateNames, Map existingPolicies, DiscoveryNodes nodes) { return createClusterChangedEvent(Settings.EMPTY, existingTemplateNames, existingPolicies, nodes); } private ClusterChangedEvent createClusterChangedEvent(Settings nodeSettings, - List existingTemplateNames, + Map existingTemplates, Map existingPolicies, DiscoveryNodes nodes) { - ClusterState cs = createClusterState(nodeSettings, existingTemplateNames, existingPolicies, nodes); + ClusterState cs = createClusterState(nodeSettings, existingTemplates, existingPolicies, nodes); ClusterChangedEvent realEvent = new ClusterChangedEvent("created-from-test", cs, ClusterState.builder(new ClusterName("test")).build()); ClusterChangedEvent event = spy(realEvent); @@ -305,10 +346,11 @@ private ClusterChangedEvent createClusterChangedEvent(Settings nodeSettings, return event; } - private ClusterState createClusterState(String ... existingTemplates) { + private ClusterState createClusterState(Map existingTemplates) { MetaData.Builder metaDataBuilder = MetaData.builder(); - for (String templateName : existingTemplates) { - metaDataBuilder.put(IndexTemplateMetaData.builder(templateName) + for (Map.Entry template : existingTemplates.entrySet()) { + metaDataBuilder.put(IndexTemplateMetaData.builder(template.getKey()) + .version(template.getValue()) .patterns(Arrays.asList(generateRandomStringArray(10, 100, false, false)))); }