Skip to content

Commit e5a0141

Browse files
martijnvgmartijnvg
committed
Added ccr to xpack usage infrastructure (#37256)
* Added ccr to xpack usage infrastructure Closes #37221
1 parent 58a76c2 commit e5a0141

File tree

9 files changed

+445
-0
lines changed

9 files changed

+445
-0
lines changed

‎docs/reference/rest-api/info.asciidoc

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,11 @@ Example response:
6363
"expiry_date_in_millis" : 1542665112332
6464
},
6565
"features" : {
66+
"ccr" : {
67+
"description" : "Cross Cluster Replication",
68+
"available" : true,
69+
"enabled" : true
70+
},
6671
"graph" : {
6772
"description" : "Graph Data Exploration for the Elastic Stack",
6873
"available" : true,

‎x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/XPackUsageIT.java

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.ccr;
7+
8+
import org.elasticsearch.client.Request;
9+
import org.elasticsearch.client.RestClient;
10+
import org.elasticsearch.common.Strings;
11+
import org.elasticsearch.common.settings.Settings;
12+
13+
import java.io.IOException;
14+
import java.util.Map;
15+
16+
import static org.hamcrest.Matchers.equalTo;
17+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
18+
import static org.hamcrest.Matchers.nullValue;
19+
20+
public class XPackUsageIT extends ESCCRRestTestCase {
21+
22+
public void testXPackCcrUsage() throws Exception {
23+
if ("follow".equals(targetCluster) == false) {
24+
logger.info("skipping test, waiting for target cluster [follow]" );
25+
return;
26+
}
27+
28+
Map<?, ?> previousUsage = getCcrUsage();
29+
putAutoFollowPattern("my_pattern", "leader_cluster", "messages-*");
30+
31+
// This index should be auto followed:
32+
createLeaderIndex("messages-20200101");
33+
// This index will be followed manually
34+
createLeaderIndex("my_index");
35+
followIndex("my_index", "my_index");
36+
37+
int previousFollowerIndicesCount = (Integer) previousUsage.get("follower_indices_count");
38+
int previousAutoFollowPatternsCount = (Integer) previousUsage.get("auto_follow_patterns_count");
39+
assertBusy(() -> {
40+
Map<?, ?> ccrUsage = getCcrUsage();
41+
assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount + 2));
42+
assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount + 1));
43+
assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0));
44+
});
45+
46+
deleteAutoFollowPattern("my_pattern");
47+
pauseFollow("messages-20200101");
48+
closeIndex("messages-20200101");
49+
unfollow("messages-20200101");
50+
51+
pauseFollow("my_index");
52+
closeIndex("my_index");
53+
unfollow("my_index");
54+
55+
assertBusy(() -> {
56+
Map<?, ?> ccrUsage = getCcrUsage();
57+
assertThat(ccrUsage.get("follower_indices_count"), equalTo(previousFollowerIndicesCount));
58+
assertThat(ccrUsage.get("auto_follow_patterns_count"), equalTo(previousAutoFollowPatternsCount));
59+
if (previousFollowerIndicesCount == 0) {
60+
assertThat(ccrUsage.get("last_follow_time_in_millis"), nullValue());
61+
} else {
62+
assertThat((Integer) ccrUsage.get("last_follow_time_in_millis"), greaterThanOrEqualTo(0));
63+
}
64+
});
65+
}
66+
67+
private void createLeaderIndex(String indexName) throws IOException {
68+
try (RestClient leaderClient = buildLeaderClient()) {
69+
Settings settings = Settings.builder()
70+
.put("index.soft_deletes.enabled", true)
71+
.build();
72+
Request request = new Request("PUT", "/" + indexName);
73+
request.setJsonEntity("{\"settings\": " + Strings.toString(settings) + "}");
74+
assertOK(leaderClient.performRequest(request));
75+
}
76+
}
77+
78+
private Map<?, ?> getCcrUsage() throws IOException {
79+
Request request = new Request("GET", "/_xpack/usage");
80+
Map<String, ?> response = toMap(client().performRequest(request));
81+
logger.info("xpack usage response={}", response);
82+
return (Map<?, ?>) response.get("ccr");
83+
}
84+
85+
}

‎x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,22 @@ protected static void pauseFollow(RestClient client, String followIndex) throws
8686
assertOK(client.performRequest(new Request("POST", "/" + followIndex + "/_ccr/pause_follow")));
8787
}
8888

89+
protected static void putAutoFollowPattern(String patternName, String remoteCluster, String indexPattern) throws IOException {
90+
Request putPatternRequest = new Request("PUT", "/_ccr/auto_follow/" + patternName);
91+
putPatternRequest.setJsonEntity("{\"leader_index_patterns\": [\"" + indexPattern + "\"], \"remote_cluster\": \"" +
92+
remoteCluster + "\"}");
93+
assertOK(client().performRequest(putPatternRequest));
94+
}
95+
96+
protected static void deleteAutoFollowPattern(String patternName) throws IOException {
97+
Request putPatternRequest = new Request("DELETE", "/_ccr/auto_follow/" + patternName);
98+
assertOK(client().performRequest(putPatternRequest));
99+
}
100+
101+
protected static void unfollow(String followIndex) throws IOException {
102+
assertOK(client().performRequest(new Request("POST", "/" + followIndex + "/_ccr/unfollow")));
103+
}
104+
89105
protected static void verifyDocuments(final String index, final int expectedNumDocs, final String query) throws IOException {
90106
verifyDocuments(index, expectedNumDocs, query, adminClient());
91107
}

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/Ccr.java

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import org.elasticsearch.cluster.node.DiscoveryNodes;
1717
import org.elasticsearch.cluster.service.ClusterService;
1818
import org.elasticsearch.common.ParseField;
19+
import org.elasticsearch.common.inject.Module;
1920
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
2021
import org.elasticsearch.common.settings.ClusterSettings;
2122
import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -83,6 +84,7 @@
8384
import org.elasticsearch.xpack.ccr.rest.RestUnfollowAction;
8485
import org.elasticsearch.xpack.core.XPackClientActionPlugin;
8586
import org.elasticsearch.xpack.core.XPackPlugin;
87+
import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
8688
import org.elasticsearch.xpack.core.ccr.ShardFollowNodeTaskStatus;
8789
import org.elasticsearch.xpack.core.ccr.action.CcrStatsAction;
8890
import org.elasticsearch.xpack.core.ccr.action.DeleteAutoFollowPatternAction;
@@ -128,6 +130,7 @@ public class Ccr extends Plugin implements ActionPlugin, PersistentTaskPlugin, E
128130
private final SetOnce<CcrRestoreSourceService> restoreSourceService = new SetOnce<>();
129131
private final SetOnce<CcrSettings> ccrSettings = new SetOnce<>();
130132
private Client client;
133+
private final boolean transportClientMode;
131134

132135
private final boolean tribeNode;
133136
private final boolean tribeNodeClient;
@@ -154,6 +157,7 @@ public Ccr(final Settings settings) {
154157
this.tribeNode = XPackClientActionPlugin.isTribeNode(settings);
155158
this.tribeNodeClient = XPackClientActionPlugin.isTribeClientNode(settings);
156159
this.ccrLicenseChecker = Objects.requireNonNull(ccrLicenseChecker);
160+
this.transportClientMode = XPackPlugin.transportClientMode(settings);
157161
}
158162

159163
@Override
@@ -321,6 +325,15 @@ public void onIndexModule(IndexModule indexModule) {
321325
}
322326
}
323327

328+
@Override
329+
public Collection<Module> createGuiceModules() {
330+
if (transportClientMode) {
331+
return Collections.emptyList();
332+
}
333+
334+
return Collections.singleton(b -> XPackPlugin.bindFeatureSet(b, CCRFeatureSet.class));
335+
}
336+
324337
protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); }
325338

326339
@Override

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetTests.java

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.ccr;
7+
8+
import org.elasticsearch.Version;
9+
import org.elasticsearch.action.support.PlainActionFuture;
10+
import org.elasticsearch.cluster.ClusterName;
11+
import org.elasticsearch.cluster.ClusterState;
12+
import org.elasticsearch.cluster.metadata.IndexMetaData;
13+
import org.elasticsearch.cluster.metadata.MetaData;
14+
import org.elasticsearch.cluster.service.ClusterService;
15+
import org.elasticsearch.common.settings.Settings;
16+
import org.elasticsearch.license.XPackLicenseState;
17+
import org.elasticsearch.test.ESTestCase;
18+
import org.elasticsearch.xpack.core.XPackFeatureSet;
19+
import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
20+
import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
21+
import org.junit.Before;
22+
import org.mockito.Mockito;
23+
24+
import java.util.Collections;
25+
import java.util.HashMap;
26+
import java.util.Map;
27+
28+
import static org.hamcrest.Matchers.equalTo;
29+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
30+
import static org.mockito.Mockito.mock;
31+
import static org.mockito.Mockito.when;
32+
33+
public class CCRFeatureSetTests extends ESTestCase {
34+
35+
private XPackLicenseState licenseState;
36+
private ClusterService clusterService;
37+
38+
@Before
39+
public void init() throws Exception {
40+
licenseState = mock(XPackLicenseState.class);
41+
clusterService = mock(ClusterService.class);
42+
}
43+
44+
public void testAvailable() {
45+
CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
46+
47+
when(licenseState.isCcrAllowed()).thenReturn(false);
48+
assertThat(featureSet.available(), equalTo(false));
49+
50+
when(licenseState.isCcrAllowed()).thenReturn(true);
51+
assertThat(featureSet.available(), equalTo(true));
52+
53+
featureSet = new CCRFeatureSet(Settings.EMPTY, null, clusterService);
54+
assertThat(featureSet.available(), equalTo(false));
55+
}
56+
57+
public void testEnabled() {
58+
Settings.Builder settings = Settings.builder().put("xpack.ccr.enabled", false);
59+
CCRFeatureSet featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
60+
assertThat(featureSet.enabled(), equalTo(false));
61+
62+
settings = Settings.builder().put("xpack.ccr.enabled", true);
63+
featureSet = new CCRFeatureSet(settings.build(), licenseState, clusterService);
64+
assertThat(featureSet.enabled(), equalTo(true));
65+
}
66+
67+
public void testName() {
68+
CCRFeatureSet featureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
69+
assertThat(featureSet.name(), equalTo("ccr"));
70+
}
71+
72+
public void testNativeCodeInfo() {
73+
CCRFeatureSet featureSet = new CCRFeatureSet (Settings.EMPTY, licenseState, clusterService);
74+
assertNull(featureSet.nativeCodeInfo());
75+
}
76+
77+
public void testUsageStats() throws Exception {
78+
MetaData.Builder metaData = MetaData.builder();
79+
80+
int numFollowerIndices = randomIntBetween(0, 32);
81+
for (int i = 0; i < numFollowerIndices; i++) {
82+
IndexMetaData.Builder followerIndex = IndexMetaData.builder("follow_index" + i)
83+
.settings(settings(Version.CURRENT).put(CcrSettings.CCR_FOLLOWING_INDEX_SETTING.getKey(), true))
84+
.numberOfShards(1)
85+
.numberOfReplicas(0)
86+
.creationDate(i)
87+
.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>());
88+
metaData.put(followerIndex);
89+
}
90+
91+
// Add a regular index, to check that we do not take that one into account:
92+
IndexMetaData.Builder regularIndex = IndexMetaData.builder("my_index")
93+
.settings(settings(Version.CURRENT))
94+
.numberOfShards(1)
95+
.numberOfReplicas(0)
96+
.creationDate(numFollowerIndices);
97+
metaData.put(regularIndex);
98+
99+
int numAutoFollowPatterns = randomIntBetween(0, 32);
100+
Map<String, AutoFollowMetadata.AutoFollowPattern> patterns = new HashMap<>(numAutoFollowPatterns);
101+
for (int i = 0; i < numAutoFollowPatterns; i++) {
102+
AutoFollowMetadata.AutoFollowPattern pattern = new AutoFollowMetadata.AutoFollowPattern("remote_cluser",
103+
Collections.singletonList("logs" + i + "*"), null, null, null, null, null, null, null, null, null, null, null);
104+
patterns.put("pattern" + i, pattern);
105+
}
106+
metaData.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap()));
107+
108+
ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metaData(metaData).build();
109+
Mockito.when(clusterService.state()).thenReturn(clusterState);
110+
111+
PlainActionFuture<XPackFeatureSet.Usage> future = new PlainActionFuture<>();
112+
CCRFeatureSet ccrFeatureSet = new CCRFeatureSet(Settings.EMPTY, licenseState, clusterService);
113+
ccrFeatureSet.usage(future);
114+
CCRFeatureSet.Usage ccrUsage = (CCRFeatureSet.Usage) future.get();
115+
assertThat(ccrUsage.enabled(), equalTo(ccrFeatureSet.enabled()));
116+
assertThat(ccrUsage.available(), equalTo(ccrFeatureSet.available()));
117+
118+
assertThat(ccrUsage.getNumberOfFollowerIndices(), equalTo(numFollowerIndices));
119+
assertThat(ccrUsage.getLastFollowTimeInMillis(), greaterThanOrEqualTo(0L));
120+
assertThat(ccrUsage.getNumberOfAutoFollowPatterns(), equalTo(numAutoFollowPatterns));
121+
}
122+
123+
}

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRFeatureSetUsageTests.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
7+
package org.elasticsearch.xpack.ccr;
8+
9+
import org.elasticsearch.common.io.stream.Writeable;
10+
import org.elasticsearch.test.AbstractWireSerializingTestCase;
11+
import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
12+
13+
public class CCRFeatureSetUsageTests extends AbstractWireSerializingTestCase<CCRFeatureSet.Usage> {
14+
15+
@Override
16+
protected CCRFeatureSet.Usage createTestInstance() {
17+
return new CCRFeatureSet.Usage(randomBoolean(), randomBoolean(), randomIntBetween(0, Integer.MAX_VALUE),
18+
randomIntBetween(0, Integer.MAX_VALUE), randomNonNegativeLong());
19+
}
20+
21+
@Override
22+
protected Writeable.Reader<CCRFeatureSet.Usage> instanceReader() {
23+
return CCRFeatureSet.Usage::new;
24+
}
25+
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import org.elasticsearch.xpack.core.action.XPackUsageAction;
4141
import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
4242
import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
43+
import org.elasticsearch.xpack.core.ccr.CCRFeatureSet;
4344
import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
4445
import org.elasticsearch.xpack.core.graph.GraphFeatureSetUsage;
4546
import org.elasticsearch.xpack.core.graph.action.GraphExploreAction;
@@ -412,6 +413,7 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
412413
new NamedWriteableRegistry.Entry(MetaData.Custom.class, AutoFollowMetadata.TYPE, AutoFollowMetadata::new),
413414
new NamedWriteableRegistry.Entry(NamedDiff.class, AutoFollowMetadata.TYPE,
414415
in -> AutoFollowMetadata.readDiffFrom(MetaData.Custom.class, AutoFollowMetadata.TYPE, in)),
416+
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.CCR, CCRFeatureSet.Usage::new),
415417
// ILM
416418
new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.INDEX_LIFECYCLE,
417419
IndexLifecycleFeatureSetUsage::new),

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ public final class XPackField {
3333
public static final String ROLLUP = "rollup";
3434
/** Name constant for the index lifecycle feature. */
3535
public static final String INDEX_LIFECYCLE = "ilm";
36+
/** Name constant for the CCR feature. */
37+
public static final String CCR = "ccr";
3638

3739
private XPackField() {}
3840

0 commit comments

Comments
 (0)