Skip to content

Commit 6ecd80c

Browse files
fcofdezfcofdez
authored
Add support for index pattern exclusion in CCR AutoFollow (#73765)
This commit adds the ability to specify exclusion patterns in Auto-Follow patterns. This allows excluding indices that match any of the inclusion patterns and also match some of the exclusion patterns giving more fine grained control in scenarios where this is important. Related #67686 Backport of #72935
1 parent a7b0e2c commit 6ecd80c

27 files changed

+577
-40
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java

+21-2
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,19 @@ public static class Pattern extends FollowConfig {
8282

8383
@SuppressWarnings("unchecked")
8484
private static final ConstructingObjectParser<Pattern, Void> PARSER = new ConstructingObjectParser<>(
85-
"pattern", true, args -> new Pattern((String) args[0], (List<String>) args[1], (String) args[2]));
85+
"pattern",
86+
true,
87+
args -> new Pattern((String) args[0],
88+
(List<String>) args[1],
89+
args[2] == null ? Collections.emptyList() : (List<String>) args[2],
90+
(String) args[3])
91+
);
8692

8793
static {
8894
PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD);
8995
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PutAutoFollowPatternRequest.LEADER_PATTERNS_FIELD);
96+
PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(),
97+
PutAutoFollowPatternRequest.LEADER_EXCLUSION_PATTERNS_FIELD);
9098
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PutAutoFollowPatternRequest.FOLLOW_PATTERN_FIELD);
9199
PARSER.declareObject(Pattern::setSettings, (p, c) -> Settings.fromXContent(p), PutAutoFollowPatternRequest.SETTINGS);
92100
PARSER.declareInt(Pattern::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT);
@@ -123,11 +131,16 @@ public static class Pattern extends FollowConfig {
123131

124132
private final String remoteCluster;
125133
private final List<String> leaderIndexPatterns;
134+
private final List<String> leaderIndexExclusionPatterns;
126135
private final String followIndexNamePattern;
127136

128-
Pattern(String remoteCluster, List<String> leaderIndexPatterns, String followIndexNamePattern) {
137+
Pattern(String remoteCluster,
138+
List<String> leaderIndexPatterns,
139+
List<String> leaderIndexExclusionPatterns,
140+
String followIndexNamePattern) {
129141
this.remoteCluster = remoteCluster;
130142
this.leaderIndexPatterns = leaderIndexPatterns;
143+
this.leaderIndexExclusionPatterns = leaderIndexExclusionPatterns;
131144
this.followIndexNamePattern = followIndexNamePattern;
132145
}
133146

@@ -139,6 +152,10 @@ public List<String> getLeaderIndexPatterns() {
139152
return leaderIndexPatterns;
140153
}
141154

155+
public List<String> getLeaderIndexExclusionPatterns() {
156+
return leaderIndexExclusionPatterns;
157+
}
158+
142159
public String getFollowIndexNamePattern() {
143160
return followIndexNamePattern;
144161
}
@@ -151,6 +168,7 @@ public boolean equals(Object o) {
151168
Pattern pattern = (Pattern) o;
152169
return Objects.equals(remoteCluster, pattern.remoteCluster) &&
153170
Objects.equals(leaderIndexPatterns, pattern.leaderIndexPatterns) &&
171+
Objects.equals(leaderIndexExclusionPatterns, pattern.leaderIndexExclusionPatterns) &&
154172
Objects.equals(followIndexNamePattern, pattern.followIndexNamePattern);
155173
}
156174

@@ -160,6 +178,7 @@ public int hashCode() {
160178
super.hashCode(),
161179
remoteCluster,
162180
leaderIndexPatterns,
181+
leaderIndexExclusionPatterns,
163182
followIndexNamePattern
164183
);
165184
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java

+23-1
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,36 @@
1414
import org.elasticsearch.common.xcontent.XContentBuilder;
1515

1616
import java.io.IOException;
17+
import java.util.Collections;
1718
import java.util.List;
1819
import java.util.Objects;
1920

2021
public final class PutAutoFollowPatternRequest extends FollowConfig implements Validatable, ToXContentObject {
2122

2223
static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns");
24+
static final ParseField LEADER_EXCLUSION_PATTERNS_FIELD = new ParseField("leader_index_exclusion_patterns");
2325
static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern");
2426

2527
private final String name;
2628
private final String remoteCluster;
2729
private final List<String> leaderIndexPatterns;
30+
private final List<String> leaderIndexExclusionPatterns;
2831
private String followIndexNamePattern;
2932

30-
public PutAutoFollowPatternRequest(String name, String remoteCluster, List<String> leaderIndexPatterns) {
33+
public PutAutoFollowPatternRequest(String name,
34+
String remoteCluster,
35+
List<String> leaderIndexPatterns) {
36+
this(name, remoteCluster, leaderIndexPatterns, Collections.emptyList());
37+
}
38+
39+
public PutAutoFollowPatternRequest(String name,
40+
String remoteCluster,
41+
List<String> leaderIndexPatterns,
42+
List<String> leaderIndexExclusionPatterns) {
3143
this.name = Objects.requireNonNull(name);
3244
this.remoteCluster = Objects.requireNonNull(remoteCluster);
3345
this.leaderIndexPatterns = Objects.requireNonNull(leaderIndexPatterns);
46+
this.leaderIndexExclusionPatterns = Objects.requireNonNull(leaderIndexExclusionPatterns);
3447
}
3548

3649
public String getName() {
@@ -45,6 +58,10 @@ public List<String> getLeaderIndexPatterns() {
4558
return leaderIndexPatterns;
4659
}
4760

61+
public List<String> getLeaderIndexExclusionPatterns() {
62+
return leaderIndexExclusionPatterns;
63+
}
64+
4865
public String getFollowIndexNamePattern() {
4966
return followIndexNamePattern;
5067
}
@@ -58,6 +75,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
5875
builder.startObject();
5976
builder.field(PutFollowRequest.REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster);
6077
builder.field(LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns);
78+
if (leaderIndexExclusionPatterns.isEmpty() == false) {
79+
builder.field(LEADER_EXCLUSION_PATTERNS_FIELD.getPreferredName(), leaderIndexExclusionPatterns);
80+
}
6181
if (followIndexNamePattern != null) {
6282
builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern);
6383
}
@@ -75,6 +95,7 @@ public boolean equals(Object o) {
7595
return Objects.equals(name, that.name) &&
7696
Objects.equals(remoteCluster, that.remoteCluster) &&
7797
Objects.equals(leaderIndexPatterns, that.leaderIndexPatterns) &&
98+
Objects.equals(leaderIndexExclusionPatterns, that.leaderIndexExclusionPatterns) &&
7899
Objects.equals(followIndexNamePattern, that.followIndexNamePattern);
79100
}
80101

@@ -85,6 +106,7 @@ public int hashCode() {
85106
name,
86107
remoteCluster,
87108
leaderIndexPatterns,
109+
leaderIndexExclusionPatterns,
88110
followIndexNamePattern
89111
);
90112
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java

+13-2
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import static org.hamcrest.Matchers.hasSize;
5959
import static org.hamcrest.Matchers.is;
6060
import static org.hamcrest.Matchers.notNullValue;
61+
import static org.hamcrest.Matchers.nullValue;
6162

6263
public class CCRIT extends ESRestHighLevelClientTestCase {
6364

@@ -245,8 +246,10 @@ public void testForgetFollower() throws IOException {
245246

246247
public void testAutoFollowing() throws Exception {
247248
CcrClient ccrClient = highLevelClient().ccr();
248-
PutAutoFollowPatternRequest putAutoFollowPatternRequest =
249-
new PutAutoFollowPatternRequest("pattern1", "local_cluster", Collections.singletonList("logs-*"));
249+
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("pattern1",
250+
"local_cluster",
251+
Collections.singletonList("logs-*"),
252+
Collections.singletonList("logs-excluded"));
250253
putAutoFollowPatternRequest.setFollowIndexNamePattern("copy-{{leader_index}}");
251254
final int followerNumberOfReplicas = randomIntBetween(0, 4);
252255
final Settings autoFollowerPatternSettings =
@@ -256,6 +259,11 @@ public void testAutoFollowing() throws Exception {
256259
execute(putAutoFollowPatternRequest, ccrClient::putAutoFollowPattern, ccrClient::putAutoFollowPatternAsync);
257260
assertThat(putAutoFollowPatternResponse.isAcknowledged(), is(true));
258261

262+
CreateIndexRequest createExcludedIndexRequest = new CreateIndexRequest("logs-excluded");
263+
CreateIndexResponse createExcludedIndexResponse =
264+
highLevelClient().indices().create(createExcludedIndexRequest, RequestOptions.DEFAULT);
265+
assertThat(createExcludedIndexResponse.isAcknowledged(), is(true));
266+
259267
CreateIndexRequest createIndexRequest = new CreateIndexRequest("logs-20200101");
260268
CreateIndexResponse response = highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
261269
assertThat(response.isAcknowledged(), is(true));
@@ -265,11 +273,13 @@ public void testAutoFollowing() throws Exception {
265273
CcrStatsResponse ccrStatsResponse = execute(ccrStatsRequest, ccrClient::getCcrStats, ccrClient::getCcrStatsAsync);
266274
assertThat(ccrStatsResponse.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), equalTo(1L));
267275
assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-20200101"), notNullValue());
276+
assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-excluded"), nullValue());
268277
});
269278
assertThat(indexExists("copy-logs-20200101"), is(true));
270279
assertThat(
271280
getIndexSettingsAsMap("copy-logs-20200101"),
272281
hasEntry("index.number_of_replicas", Integer.toString(followerNumberOfReplicas)));
282+
assertThat(indexExists("copy-logs-excluded"), is(false));
273283

274284
GetAutoFollowPatternRequest getAutoFollowPatternRequest =
275285
randomBoolean() ? new GetAutoFollowPatternRequest("pattern1") : new GetAutoFollowPatternRequest();
@@ -280,6 +290,7 @@ public void testAutoFollowing() throws Exception {
280290
assertThat(pattern, notNullValue());
281291
assertThat(pattern.getRemoteCluster(), equalTo(putAutoFollowPatternRequest.getRemoteCluster()));
282292
assertThat(pattern.getLeaderIndexPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexPatterns()));
293+
assertThat(pattern.getLeaderIndexExclusionPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexExclusionPatterns()));
283294
assertThat(pattern.getFollowIndexNamePattern(), equalTo(putAutoFollowPatternRequest.getFollowIndexNamePattern()));
284295
assertThat(pattern.getSettings(), equalTo(autoFollowerPatternSettings));
285296

‎client/rest-high-level/src/test/java/org/elasticsearch/client/CcrRequestConvertersTests.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,10 @@ public void testForgetFollower() throws IOException {
101101

102102
public void testPutAutofollowPattern() throws Exception {
103103
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest(randomAlphaOfLength(4),
104-
randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
104+
randomAlphaOfLength(4),
105+
Arrays.asList(generateRandomStringArray(4, 4, false)),
106+
Arrays.asList(generateRandomStringArray(4, 4, false))
107+
);
105108
if (randomBoolean()) {
106109
putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
107110
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java

+3
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
3939
for (int i = 0; i < numPatterns; i++) {
4040
String remoteCluster = randomAlphaOfLength(4);
4141
List<String> leaderIndexPatterns = Collections.singletonList(randomAlphaOfLength(4));
42+
List<String> leaderIndexExclusionsPatterns = randomList(0, randomIntBetween(1, 10), () -> randomAlphaOfLength(4));
4243
String followIndexNamePattern = randomAlphaOfLength(4);
4344
final Settings settings =
4445
Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build();
@@ -89,6 +90,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
8990
new AutoFollowMetadata.AutoFollowPattern(
9091
remoteCluster,
9192
leaderIndexPatterns,
93+
leaderIndexExclusionsPatterns,
9294
followIndexNamePattern,
9395
settings,
9496
active,
@@ -124,6 +126,7 @@ protected void assertInstances(GetAutoFollowPatternAction.Response serverTestIns
124126
assertThat(serverPattern.getRemoteCluster(), equalTo(clientPattern.getRemoteCluster()));
125127
assertThat(serverPattern.getLeaderIndexPatterns(), equalTo(clientPattern.getLeaderIndexPatterns()));
126128
assertThat(serverPattern.getFollowIndexPattern(), equalTo(clientPattern.getFollowIndexNamePattern()));
129+
assertThat(serverPattern.getLeaderIndexExclusionPatterns(), equalTo(clientPattern.getLeaderIndexExclusionPatterns()));
127130
assertThat(serverPattern.getSettings(), equalTo(clientPattern.getSettings()));
128131
assertThat(serverPattern.getMaxOutstandingReadRequests(), equalTo(clientPattern.getMaxOutstandingReadRequests()));
129132
assertThat(serverPattern.getMaxOutstandingWriteRequests(), equalTo(clientPattern.getMaxOutstandingWriteRequests()));

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ public class PutAutoFollowPatternRequestTests extends AbstractRequestTestCase<
2828
protected PutAutoFollowPatternRequest createClientTestInstance() {
2929
// Name isn't serialized, because it specified in url path, so no need to randomly generate it here.
3030
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("name",
31-
randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
31+
randomAlphaOfLength(4),
32+
Arrays.asList(generateRandomStringArray(4, 4, false)),
33+
Arrays.asList(generateRandomStringArray(4, 4, false))
34+
);
3235
if (randomBoolean()) {
3336
putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
3437
}
@@ -75,6 +78,7 @@ protected void assertInstances(PutAutoFollowPatternAction.Request serverInstance
7578
assertThat(serverInstance.getName(), equalTo(clientTestInstance.getName()));
7679
assertThat(serverInstance.getRemoteCluster(), equalTo(clientTestInstance.getRemoteCluster()));
7780
assertThat(serverInstance.getLeaderIndexPatterns(), equalTo(clientTestInstance.getLeaderIndexPatterns()));
81+
assertThat(serverInstance.getLeaderIndexExclusionPatterns(), equalTo(clientTestInstance.getLeaderIndexExclusionPatterns()));
7882
assertThat(serverInstance.getFollowIndexNamePattern(), equalTo(clientTestInstance.getFollowIndexNamePattern()));
7983
assertFollowConfig(serverInstance.getParameters(), clientTestInstance);
8084
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -476,12 +476,13 @@ public void testPutAutoFollowPattern() throws Exception {
476476
new PutAutoFollowPatternRequest(
477477
"my_pattern", // <1>
478478
"local", // <2>
479-
Arrays.asList("logs-*", "metrics-*") // <3>
479+
Arrays.asList("logs-*", "metrics-*"), // <3>
480+
Arrays.asList("logs-excluded", "metrics-excluded") // <4>
480481
);
481-
request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <4>
482+
request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <5>
482483
Settings settings =
483484
Settings.builder().put("index.number_of_replicas", 0L).build();
484-
request.setSettings(settings); // <5>
485+
request.setSettings(settings); // <6>
485486
// end::ccr-put-auto-follow-pattern-request
486487

487488
// tag::ccr-put-auto-follow-pattern-execute

‎docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc

+3-2
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@ include-tagged::{doc-tests-file}[{api}-request]
2121
<1> The name of the auto follow pattern.
2222
<2> The name of the remote cluster.
2323
<3> The leader index patterns.
24-
<4> The pattern used to create the follower index
25-
<5> The settings overrides for the follower index
24+
<4> The leader index exclusion patterns.
25+
<5> The pattern used to create the follower index.
26+
<6> The settings overrides for the follower index.
2627

2728
[id="{upid}-{api}-response"]
2829
==== Response

‎docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc

+8
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ PUT /_ccr/auto_follow/my_auto_follow_pattern
2222
[
2323
"leader_index*"
2424
],
25+
"leader_index_exclusion_patterns":
26+
[
27+
"leader_index_001"
28+
],
2529
"follow_index_pattern" : "{{leader_index}}-follower"
2630
}
2731
--------------------------------------------------
@@ -96,6 +100,10 @@ The API returns the following result:
96100
[
97101
"leader_index*"
98102
],
103+
"leader_index_exclusion_patterns":
104+
[
105+
"leader_index_001"
106+
],
99107
"follow_index_pattern" : "{{leader_index}}-follower"
100108
}
101109
}

‎docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc

+14-1
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,18 @@ PUT /_ccr/auto_follow/<auto_follow_pattern_name>
2020
[
2121
"<leader_index_pattern>"
2222
],
23+
"leader_index_exclusion_patterns":
24+
[
25+
"<leader_index_exclusion_pattern>"
26+
],
2327
"follow_index_pattern" : "<follow_index_pattern>"
2428
}
2529
--------------------------------------------------
2630
// TEST[setup:remote_cluster]
2731
// TEST[s/<auto_follow_pattern_name>/auto_follow_pattern_name/]
2832
// TEST[s/<remote_cluster>/remote_cluster/]
2933
// TEST[s/<leader_index_patterns>/leader_index*/]
34+
// TEST[s/<leader_index_exclusion_pattern>//]
3035
// TEST[s/<follow_index_pattern>/{{leader_index}}-follower/]
3136

3237
//////////////////////////
@@ -54,7 +59,10 @@ This API creates a new named collection of
5459
<<ccr-auto-follow,auto-follow patterns>> against the remote cluster
5560
specified in the request body. Newly created indices on the remote cluster
5661
matching any of the specified patterns will be automatically configured as follower
57-
indices.
62+
indices. Additionally, this API can be used to update existing
63+
<<ccr-auto-follow,auto-follow patterns>>. Note that follower indices that were configured automatically
64+
before updating an auto-follow pattern will remain unchanged even if they don't match against
65+
the new patterns.
5866

5967
[[ccr-put-auto-follow-pattern-path-parms]]
6068
==== {api-path-parms-title}
@@ -72,6 +80,11 @@ indices.
7280
(Optional, array) An array of simple index patterns to match against indices
7381
in the remote cluster specified by the `remote_cluster` field.
7482

83+
`leader_index_exclusion_patterns`::
84+
(Optional, array) An array of simple index patterns that can be used to exclude indices
85+
from being auto-followed. Indices in the remote cluster whose names are matching one or
86+
more `leader_index_patterns` and one or more `leader_index_exclusion_patterns` won't be followed.
87+
7588
`follow_index_pattern`::
7689
(Optional, string) The name of follower index. The template `{{leader_index}}`
7790
can be used to derive the name of the follower index from the name of the

0 commit comments

Comments
 (0)