Skip to content

Commit 9ec9685

Browse files
fcofdezfcofdez
committed
Add support for index pattern exclusion in CCR AutoFollow
1 parent 0a0fc8a commit 9ec9685

27 files changed

+483
-35
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,19 @@ public static class Pattern extends FollowConfig {
8282

8383
@SuppressWarnings("unchecked")
8484
private static final ConstructingObjectParser<Pattern, Void> PARSER = new ConstructingObjectParser<>(
85-
"pattern", true, args -> new Pattern((String) args[0], (List<String>) args[1], (String) args[2]));
85+
"pattern",
86+
true,
87+
args -> new Pattern((String) args[0],
88+
(List<String>) args[1],
89+
args[2] == null ? Collections.emptyList() : (List<String>) args[2],
90+
(String) args[3])
91+
);
8692

8793
static {
8894
PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD);
8995
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PutAutoFollowPatternRequest.LEADER_PATTERNS_FIELD);
96+
PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(),
97+
PutAutoFollowPatternRequest.LEADER_EXCLUSION_PATTERNS_FIELD);
9098
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PutAutoFollowPatternRequest.FOLLOW_PATTERN_FIELD);
9199
PARSER.declareObject(Pattern::setSettings, (p, c) -> Settings.fromXContent(p), PutAutoFollowPatternRequest.SETTINGS);
92100
PARSER.declareInt(Pattern::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT);
@@ -123,11 +131,16 @@ public static class Pattern extends FollowConfig {
123131

124132
private final String remoteCluster;
125133
private final List<String> leaderIndexPatterns;
134+
private final List<String> leaderIndexExclusionPatterns;
126135
private final String followIndexNamePattern;
127136

128-
Pattern(String remoteCluster, List<String> leaderIndexPatterns, String followIndexNamePattern) {
137+
Pattern(String remoteCluster,
138+
List<String> leaderIndexPatterns,
139+
List<String> leaderIndexExclusionPatterns,
140+
String followIndexNamePattern) {
129141
this.remoteCluster = remoteCluster;
130142
this.leaderIndexPatterns = leaderIndexPatterns;
143+
this.leaderIndexExclusionPatterns = leaderIndexExclusionPatterns;
131144
this.followIndexNamePattern = followIndexNamePattern;
132145
}
133146

@@ -139,6 +152,10 @@ public List<String> getLeaderIndexPatterns() {
139152
return leaderIndexPatterns;
140153
}
141154

155+
public List<String> getLeaderIndexExclusionPatterns() {
156+
return leaderIndexExclusionPatterns;
157+
}
158+
142159
public String getFollowIndexNamePattern() {
143160
return followIndexNamePattern;
144161
}
@@ -151,6 +168,7 @@ public boolean equals(Object o) {
151168
Pattern pattern = (Pattern) o;
152169
return Objects.equals(remoteCluster, pattern.remoteCluster) &&
153170
Objects.equals(leaderIndexPatterns, pattern.leaderIndexPatterns) &&
171+
Objects.equals(leaderIndexExclusionPatterns, pattern.leaderIndexExclusionPatterns) &&
154172
Objects.equals(followIndexNamePattern, pattern.followIndexNamePattern);
155173
}
156174

@@ -160,6 +178,7 @@ public int hashCode() {
160178
super.hashCode(),
161179
remoteCluster,
162180
leaderIndexPatterns,
181+
leaderIndexExclusionPatterns,
163182
followIndexNamePattern
164183
);
165184
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,36 @@
1414
import org.elasticsearch.common.xcontent.XContentBuilder;
1515

1616
import java.io.IOException;
17+
import java.util.Collections;
1718
import java.util.List;
1819
import java.util.Objects;
1920

2021
public final class PutAutoFollowPatternRequest extends FollowConfig implements Validatable, ToXContentObject {
2122

2223
static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns");
24+
static final ParseField LEADER_EXCLUSION_PATTERNS_FIELD = new ParseField("leader_index_exclusion_patterns");
2325
static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern");
2426

2527
private final String name;
2628
private final String remoteCluster;
2729
private final List<String> leaderIndexPatterns;
30+
private final List<String> leaderIndexExclusionPatterns;
2831
private String followIndexNamePattern;
2932

30-
public PutAutoFollowPatternRequest(String name, String remoteCluster, List<String> leaderIndexPatterns) {
33+
public PutAutoFollowPatternRequest(String name,
34+
String remoteCluster,
35+
List<String> leaderIndexPatterns) {
36+
this(name, remoteCluster, leaderIndexPatterns, Collections.emptyList());
37+
}
38+
39+
public PutAutoFollowPatternRequest(String name,
40+
String remoteCluster,
41+
List<String> leaderIndexPatterns,
42+
List<String> leaderIndexExclusionPatterns) {
3143
this.name = Objects.requireNonNull(name);
3244
this.remoteCluster = Objects.requireNonNull(remoteCluster);
3345
this.leaderIndexPatterns = Objects.requireNonNull(leaderIndexPatterns);
46+
this.leaderIndexExclusionPatterns = Objects.requireNonNull(leaderIndexExclusionPatterns);
3447
}
3548

3649
public String getName() {
@@ -45,6 +58,10 @@ public List<String> getLeaderIndexPatterns() {
4558
return leaderIndexPatterns;
4659
}
4760

61+
public List<String> getLeaderIndexExclusionPatterns() {
62+
return leaderIndexExclusionPatterns;
63+
}
64+
4865
public String getFollowIndexNamePattern() {
4966
return followIndexNamePattern;
5067
}
@@ -58,6 +75,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
5875
builder.startObject();
5976
builder.field(PutFollowRequest.REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster);
6077
builder.field(LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns);
78+
builder.field(LEADER_EXCLUSION_PATTERNS_FIELD.getPreferredName(), leaderIndexExclusionPatterns);
6179
if (followIndexNamePattern != null) {
6280
builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern);
6381
}
@@ -75,6 +93,7 @@ public boolean equals(Object o) {
7593
return Objects.equals(name, that.name) &&
7694
Objects.equals(remoteCluster, that.remoteCluster) &&
7795
Objects.equals(leaderIndexPatterns, that.leaderIndexPatterns) &&
96+
Objects.equals(leaderIndexExclusionPatterns, that.leaderIndexExclusionPatterns) &&
7897
Objects.equals(followIndexNamePattern, that.followIndexNamePattern);
7998
}
8099

@@ -85,6 +104,7 @@ public int hashCode() {
85104
name,
86105
remoteCluster,
87106
leaderIndexPatterns,
107+
leaderIndexExclusionPatterns,
88108
followIndexNamePattern
89109
);
90110
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,8 +244,10 @@ public void testForgetFollower() throws IOException {
244244

245245
public void testAutoFollowing() throws Exception {
246246
CcrClient ccrClient = highLevelClient().ccr();
247-
PutAutoFollowPatternRequest putAutoFollowPatternRequest =
248-
new PutAutoFollowPatternRequest("pattern1", "local_cluster", Collections.singletonList("logs-*"));
247+
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("pattern1",
248+
"local_cluster",
249+
Collections.singletonList("logs-*"),
250+
Collections.singletonList("logs-excluded"));
249251
putAutoFollowPatternRequest.setFollowIndexNamePattern("copy-{{leader_index}}");
250252
final int followerNumberOfReplicas = randomIntBetween(0, 4);
251253
final Settings autoFollowerPatternSettings =
@@ -270,6 +272,20 @@ public void testAutoFollowing() throws Exception {
270272
getIndexSettingsAsMap("copy-logs-20200101"),
271273
hasEntry("index.number_of_replicas", Integer.toString(followerNumberOfReplicas)));
272274

275+
CreateIndexRequest createExcludedIndexRequest = new CreateIndexRequest("logs-excluded");
276+
CreateIndexResponse createExcludedIndexResponse =
277+
highLevelClient().indices().create(createExcludedIndexRequest, RequestOptions.DEFAULT);
278+
assertThat(createExcludedIndexResponse.isAcknowledged(), is(true));
279+
280+
assertBusy(() -> {
281+
CcrStatsRequest ccrStatsRequest = new CcrStatsRequest();
282+
CcrStatsResponse ccrStatsResponse = execute(ccrStatsRequest, ccrClient::getCcrStats, ccrClient::getCcrStatsAsync);
283+
assertThat(ccrStatsResponse.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), equalTo(1L));
284+
assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-20200101"), notNullValue());
285+
});
286+
287+
assertThat(indexExists("copy-logs-excluded"), is(false));
288+
273289
GetAutoFollowPatternRequest getAutoFollowPatternRequest =
274290
randomBoolean() ? new GetAutoFollowPatternRequest("pattern1") : new GetAutoFollowPatternRequest();
275291
GetAutoFollowPatternResponse getAutoFollowPatternResponse =
@@ -279,6 +295,7 @@ public void testAutoFollowing() throws Exception {
279295
assertThat(pattern, notNullValue());
280296
assertThat(pattern.getRemoteCluster(), equalTo(putAutoFollowPatternRequest.getRemoteCluster()));
281297
assertThat(pattern.getLeaderIndexPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexPatterns()));
298+
assertThat(pattern.getLeaderIndexExclusionPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexExclusionPatterns()));
282299
assertThat(pattern.getFollowIndexNamePattern(), equalTo(putAutoFollowPatternRequest.getFollowIndexNamePattern()));
283300
assertThat(pattern.getSettings(), equalTo(autoFollowerPatternSettings));
284301

‎client/rest-high-level/src/test/java/org/elasticsearch/client/CcrRequestConvertersTests.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,10 @@ public void testForgetFollower() throws IOException {
101101

102102
public void testPutAutofollowPattern() throws Exception {
103103
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest(randomAlphaOfLength(4),
104-
randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
104+
randomAlphaOfLength(4),
105+
Arrays.asList(generateRandomStringArray(4, 4, false)),
106+
Arrays.asList(generateRandomStringArray(4, 4, false))
107+
);
105108
if (randomBoolean()) {
106109
putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
107110
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
3939
for (int i = 0; i < numPatterns; i++) {
4040
String remoteCluster = randomAlphaOfLength(4);
4141
List<String> leaderIndexPatterns = Collections.singletonList(randomAlphaOfLength(4));
42+
List<String> leaderIndexExclusionsPatterns = Collections.singletonList(randomAlphaOfLength(4));
4243
String followIndexNamePattern = randomAlphaOfLength(4);
4344
final Settings settings =
4445
Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build();
@@ -89,6 +90,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
8990
new AutoFollowMetadata.AutoFollowPattern(
9091
remoteCluster,
9192
leaderIndexPatterns,
93+
leaderIndexExclusionsPatterns,
9294
followIndexNamePattern,
9395
settings,
9496
active,
@@ -124,6 +126,7 @@ protected void assertInstances(GetAutoFollowPatternAction.Response serverTestIns
124126
assertThat(serverPattern.getRemoteCluster(), equalTo(clientPattern.getRemoteCluster()));
125127
assertThat(serverPattern.getLeaderIndexPatterns(), equalTo(clientPattern.getLeaderIndexPatterns()));
126128
assertThat(serverPattern.getFollowIndexPattern(), equalTo(clientPattern.getFollowIndexNamePattern()));
129+
assertThat(serverPattern.getLeaderIndexExclusionPatterns(), equalTo(clientPattern.getLeaderIndexExclusionPatterns()));
127130
assertThat(serverPattern.getSettings(), equalTo(clientPattern.getSettings()));
128131
assertThat(serverPattern.getMaxOutstandingReadRequests(), equalTo(clientPattern.getMaxOutstandingReadRequests()));
129132
assertThat(serverPattern.getMaxOutstandingWriteRequests(), equalTo(clientPattern.getMaxOutstandingWriteRequests()));

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ public class PutAutoFollowPatternRequestTests extends AbstractRequestTestCase<
2828
protected PutAutoFollowPatternRequest createClientTestInstance() {
2929
// Name isn't serialized, because it specified in url path, so no need to randomly generate it here.
3030
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("name",
31-
randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
31+
randomAlphaOfLength(4),
32+
Arrays.asList(generateRandomStringArray(4, 4, false)),
33+
Arrays.asList(generateRandomStringArray(4, 4, false))
34+
);
3235
if (randomBoolean()) {
3336
putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
3437
}
@@ -75,6 +78,7 @@ protected void assertInstances(PutAutoFollowPatternAction.Request serverInstance
7578
assertThat(serverInstance.getName(), equalTo(clientTestInstance.getName()));
7679
assertThat(serverInstance.getRemoteCluster(), equalTo(clientTestInstance.getRemoteCluster()));
7780
assertThat(serverInstance.getLeaderIndexPatterns(), equalTo(clientTestInstance.getLeaderIndexPatterns()));
81+
assertThat(serverInstance.getLeaderIndexExclusionPatterns(), equalTo(clientTestInstance.getLeaderIndexExclusionPatterns()));
7882
assertThat(serverInstance.getFollowIndexNamePattern(), equalTo(clientTestInstance.getFollowIndexNamePattern()));
7983
assertFollowConfig(serverInstance.getParameters(), clientTestInstance);
8084
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -474,12 +474,13 @@ public void testPutAutoFollowPattern() throws Exception {
474474
new PutAutoFollowPatternRequest(
475475
"my_pattern", // <1>
476476
"local", // <2>
477-
Arrays.asList("logs-*", "metrics-*") // <3>
477+
Arrays.asList("logs-*", "metrics-*"), // <3>
478+
Arrays.asList("logs-excluded", "metrics-excluded") // <4>
478479
);
479-
request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <4>
480+
request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <5>
480481
Settings settings =
481482
Settings.builder().put("index.number_of_replicas", 0L).build();
482-
request.setSettings(settings); // <5>
483+
request.setSettings(settings); // <6>
483484
// end::ccr-put-auto-follow-pattern-request
484485

485486
// tag::ccr-put-auto-follow-pattern-execute

‎docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@ include-tagged::{doc-tests-file}[{api}-request]
2121
<1> The name of the auto follow pattern.
2222
<2> The name of the remote cluster.
2323
<3> The leader index patterns.
24-
<4> The pattern used to create the follower index
25-
<5> The settings overrides for the follower index
24+
<4> The leader index exclusion patterns.
25+
<5> The pattern used to create the follower index.
26+
<6> The settings overrides for the follower index.
2627

2728
[id="{upid}-{api}-response"]
2829
==== Response

‎docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ The API returns the following result:
9696
[
9797
"leader_index*"
9898
],
99+
"leader_index_exclusion_patterns": [],
99100
"follow_index_pattern" : "{{leader_index}}-follower"
100101
}
101102
}

‎docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,18 @@ PUT /_ccr/auto_follow/<auto_follow_pattern_name>
2020
[
2121
"<leader_index_pattern>"
2222
],
23+
"leader_index_exclusion_patterns":
24+
[
25+
"leader_index_exclusion_pattern>"
26+
],
2327
"follow_index_pattern" : "<follow_index_pattern>"
2428
}
2529
--------------------------------------------------
2630
// TEST[setup:remote_cluster]
2731
// TEST[s/<auto_follow_pattern_name>/auto_follow_pattern_name/]
2832
// TEST[s/<remote_cluster>/remote_cluster/]
2933
// TEST[s/<leader_index_patterns>/leader_index*/]
34+
// TEST[s/<leader_index_exclusion_pattern>//]
3035
// TEST[s/<follow_index_pattern>/{{leader_index}}-follower/]
3136

3237
//////////////////////////
@@ -72,6 +77,10 @@ indices.
7277
(Optional, array) An array of simple index patterns to match against indices
7378
in the remote cluster specified by the `remote_cluster` field.
7479

80+
`leader_index_exclusion_patterns`::
81+
(Optional, array) An array of simple index patterns to avoid following
82+
indices in the remote cluster specified by the `remote_cluster` field.
83+
7584
`follow_index_pattern`::
7685
(Optional, string) The name of follower index. The template `{{leader_index}}`
7786
can be used to derive the name of the follower index from the name of the

‎x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,64 @@ public void testAutoFollowPatterns() throws Exception {
174174
}
175175
}
176176

177+
public void testAutoFollowExclusionPatterns() throws Exception {
178+
if ("follow".equals(targetCluster) == false) {
179+
logger.info("skipping test, waiting for target cluster [follow]");
180+
return;
181+
}
182+
183+
final String autoFollowPatternName = getTestName().toLowerCase(Locale.ROOT);
184+
try {
185+
final String excludedIndex = "metrics-20220102";
186+
int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices();
187+
Request request = new Request("PUT", "/_ccr/auto_follow/" + autoFollowPatternName);
188+
try (XContentBuilder bodyBuilder = JsonXContent.contentBuilder()) {
189+
bodyBuilder.startObject();
190+
{
191+
bodyBuilder.startArray("leader_index_patterns");
192+
{
193+
bodyBuilder.value("metrics-*");
194+
}
195+
bodyBuilder.endArray();
196+
bodyBuilder.startArray("leader_index_exclusion_patterns");
197+
{
198+
bodyBuilder.value(excludedIndex);
199+
}
200+
bodyBuilder.endArray();
201+
bodyBuilder.field("remote_cluster", "leader_cluster");
202+
}
203+
bodyBuilder.endObject();
204+
request.setJsonEntity(Strings.toString(bodyBuilder));
205+
}
206+
assertOK(client().performRequest(request));
207+
208+
try (RestClient leaderClient = buildLeaderClient()) {
209+
request = new Request("PUT", "/metrics-20220101");
210+
request.setJsonEntity("{\"mappings\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}");
211+
assertOK(leaderClient.performRequest(request));
212+
}
213+
214+
assertBusy(() -> {
215+
assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1));
216+
ensureYellow("metrics-20220101");
217+
});
218+
219+
try (RestClient leaderClient = buildLeaderClient()) {
220+
request = new Request("PUT", "/" + excludedIndex);
221+
request.setJsonEntity("{\"mappings\": {\"properties\": {\"field\": {\"type\": \"keyword\"}}}}");
222+
assertOK(leaderClient.performRequest(request));
223+
}
224+
225+
assertBusy(() -> {
226+
assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1));
227+
});
228+
229+
assertThat(indexExists(excludedIndex), is(false));
230+
} finally {
231+
cleanUpFollower(List.of("metrics-20220101"), List.of(), List.of(autoFollowPatternName));
232+
}
233+
}
234+
177235
public void testPutAutoFollowPatternThatOverridesRequiredLeaderSetting() throws IOException {
178236
if ("follow".equals(targetCluster) == false) {
179237
logger.info("skipping test, waiting for target cluster [follow]");

0 commit comments

Comments
 (0)