Skip to content

Commit a8101f3

Browse files
fcofdezfcofdez
authored
Add support for index pattern exclusion in CCR AutoFollow (#72935)
This commit adds the ability to specify exclusion patterns in Auto-Follow patterns. This allows excluding indices that match any of the inclusion patterns and also match some of the exclusion patterns giving more fine grained control in scenarios where this is important. Related #67686
1 parent 9a8be68 commit a8101f3

27 files changed

+570
-40
lines changed

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponse.java

+21-2
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,19 @@ public static class Pattern extends FollowConfig {
8282

8383
@SuppressWarnings("unchecked")
8484
private static final ConstructingObjectParser<Pattern, Void> PARSER = new ConstructingObjectParser<>(
85-
"pattern", true, args -> new Pattern((String) args[0], (List<String>) args[1], (String) args[2]));
85+
"pattern",
86+
true,
87+
args -> new Pattern((String) args[0],
88+
(List<String>) args[1],
89+
args[2] == null ? Collections.emptyList() : (List<String>) args[2],
90+
(String) args[3])
91+
);
8692

8793
static {
8894
PARSER.declareString(ConstructingObjectParser.constructorArg(), PutFollowRequest.REMOTE_CLUSTER_FIELD);
8995
PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), PutAutoFollowPatternRequest.LEADER_PATTERNS_FIELD);
96+
PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(),
97+
PutAutoFollowPatternRequest.LEADER_EXCLUSION_PATTERNS_FIELD);
9098
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), PutAutoFollowPatternRequest.FOLLOW_PATTERN_FIELD);
9199
PARSER.declareObject(Pattern::setSettings, (p, c) -> Settings.fromXContent(p), PutAutoFollowPatternRequest.SETTINGS);
92100
PARSER.declareInt(Pattern::setMaxReadRequestOperationCount, FollowConfig.MAX_READ_REQUEST_OPERATION_COUNT);
@@ -123,11 +131,16 @@ public static class Pattern extends FollowConfig {
123131

124132
private final String remoteCluster;
125133
private final List<String> leaderIndexPatterns;
134+
private final List<String> leaderIndexExclusionPatterns;
126135
private final String followIndexNamePattern;
127136

128-
Pattern(String remoteCluster, List<String> leaderIndexPatterns, String followIndexNamePattern) {
137+
Pattern(String remoteCluster,
138+
List<String> leaderIndexPatterns,
139+
List<String> leaderIndexExclusionPatterns,
140+
String followIndexNamePattern) {
129141
this.remoteCluster = remoteCluster;
130142
this.leaderIndexPatterns = leaderIndexPatterns;
143+
this.leaderIndexExclusionPatterns = leaderIndexExclusionPatterns;
131144
this.followIndexNamePattern = followIndexNamePattern;
132145
}
133146

@@ -139,6 +152,10 @@ public List<String> getLeaderIndexPatterns() {
139152
return leaderIndexPatterns;
140153
}
141154

155+
public List<String> getLeaderIndexExclusionPatterns() {
156+
return leaderIndexExclusionPatterns;
157+
}
158+
142159
public String getFollowIndexNamePattern() {
143160
return followIndexNamePattern;
144161
}
@@ -151,6 +168,7 @@ public boolean equals(Object o) {
151168
Pattern pattern = (Pattern) o;
152169
return Objects.equals(remoteCluster, pattern.remoteCluster) &&
153170
Objects.equals(leaderIndexPatterns, pattern.leaderIndexPatterns) &&
171+
Objects.equals(leaderIndexExclusionPatterns, pattern.leaderIndexExclusionPatterns) &&
154172
Objects.equals(followIndexNamePattern, pattern.followIndexNamePattern);
155173
}
156174

@@ -160,6 +178,7 @@ public int hashCode() {
160178
super.hashCode(),
161179
remoteCluster,
162180
leaderIndexPatterns,
181+
leaderIndexExclusionPatterns,
163182
followIndexNamePattern
164183
);
165184
}

‎client/rest-high-level/src/main/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequest.java

+23-1
Original file line numberDiff line numberDiff line change
@@ -14,23 +14,36 @@
1414
import org.elasticsearch.common.xcontent.XContentBuilder;
1515

1616
import java.io.IOException;
17+
import java.util.Collections;
1718
import java.util.List;
1819
import java.util.Objects;
1920

2021
public final class PutAutoFollowPatternRequest extends FollowConfig implements Validatable, ToXContentObject {
2122

2223
static final ParseField LEADER_PATTERNS_FIELD = new ParseField("leader_index_patterns");
24+
static final ParseField LEADER_EXCLUSION_PATTERNS_FIELD = new ParseField("leader_index_exclusion_patterns");
2325
static final ParseField FOLLOW_PATTERN_FIELD = new ParseField("follow_index_pattern");
2426

2527
private final String name;
2628
private final String remoteCluster;
2729
private final List<String> leaderIndexPatterns;
30+
private final List<String> leaderIndexExclusionPatterns;
2831
private String followIndexNamePattern;
2932

30-
public PutAutoFollowPatternRequest(String name, String remoteCluster, List<String> leaderIndexPatterns) {
33+
public PutAutoFollowPatternRequest(String name,
34+
String remoteCluster,
35+
List<String> leaderIndexPatterns) {
36+
this(name, remoteCluster, leaderIndexPatterns, Collections.emptyList());
37+
}
38+
39+
public PutAutoFollowPatternRequest(String name,
40+
String remoteCluster,
41+
List<String> leaderIndexPatterns,
42+
List<String> leaderIndexExclusionPatterns) {
3143
this.name = Objects.requireNonNull(name);
3244
this.remoteCluster = Objects.requireNonNull(remoteCluster);
3345
this.leaderIndexPatterns = Objects.requireNonNull(leaderIndexPatterns);
46+
this.leaderIndexExclusionPatterns = Objects.requireNonNull(leaderIndexExclusionPatterns);
3447
}
3548

3649
public String getName() {
@@ -45,6 +58,10 @@ public List<String> getLeaderIndexPatterns() {
4558
return leaderIndexPatterns;
4659
}
4760

61+
public List<String> getLeaderIndexExclusionPatterns() {
62+
return leaderIndexExclusionPatterns;
63+
}
64+
4865
public String getFollowIndexNamePattern() {
4966
return followIndexNamePattern;
5067
}
@@ -58,6 +75,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
5875
builder.startObject();
5976
builder.field(PutFollowRequest.REMOTE_CLUSTER_FIELD.getPreferredName(), remoteCluster);
6077
builder.field(LEADER_PATTERNS_FIELD.getPreferredName(), leaderIndexPatterns);
78+
if (leaderIndexExclusionPatterns.isEmpty() == false) {
79+
builder.field(LEADER_EXCLUSION_PATTERNS_FIELD.getPreferredName(), leaderIndexExclusionPatterns);
80+
}
6181
if (followIndexNamePattern != null) {
6282
builder.field(FOLLOW_PATTERN_FIELD.getPreferredName(), followIndexNamePattern);
6383
}
@@ -75,6 +95,7 @@ public boolean equals(Object o) {
7595
return Objects.equals(name, that.name) &&
7696
Objects.equals(remoteCluster, that.remoteCluster) &&
7797
Objects.equals(leaderIndexPatterns, that.leaderIndexPatterns) &&
98+
Objects.equals(leaderIndexExclusionPatterns, that.leaderIndexExclusionPatterns) &&
7899
Objects.equals(followIndexNamePattern, that.followIndexNamePattern);
79100
}
80101

@@ -85,6 +106,7 @@ public int hashCode() {
85106
name,
86107
remoteCluster,
87108
leaderIndexPatterns,
109+
leaderIndexExclusionPatterns,
88110
followIndexNamePattern
89111
);
90112
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/CCRIT.java

+13-2
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import static org.hamcrest.Matchers.hasSize;
5959
import static org.hamcrest.Matchers.is;
6060
import static org.hamcrest.Matchers.notNullValue;
61+
import static org.hamcrest.Matchers.nullValue;
6162

6263
public class CCRIT extends ESRestHighLevelClientTestCase {
6364

@@ -244,8 +245,10 @@ public void testForgetFollower() throws IOException {
244245

245246
public void testAutoFollowing() throws Exception {
246247
CcrClient ccrClient = highLevelClient().ccr();
247-
PutAutoFollowPatternRequest putAutoFollowPatternRequest =
248-
new PutAutoFollowPatternRequest("pattern1", "local_cluster", Collections.singletonList("logs-*"));
248+
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("pattern1",
249+
"local_cluster",
250+
Collections.singletonList("logs-*"),
251+
Collections.singletonList("logs-excluded"));
249252
putAutoFollowPatternRequest.setFollowIndexNamePattern("copy-{{leader_index}}");
250253
final int followerNumberOfReplicas = randomIntBetween(0, 4);
251254
final Settings autoFollowerPatternSettings =
@@ -255,6 +258,11 @@ public void testAutoFollowing() throws Exception {
255258
execute(putAutoFollowPatternRequest, ccrClient::putAutoFollowPattern, ccrClient::putAutoFollowPatternAsync);
256259
assertThat(putAutoFollowPatternResponse.isAcknowledged(), is(true));
257260

261+
CreateIndexRequest createExcludedIndexRequest = new CreateIndexRequest("logs-excluded");
262+
CreateIndexResponse createExcludedIndexResponse =
263+
highLevelClient().indices().create(createExcludedIndexRequest, RequestOptions.DEFAULT);
264+
assertThat(createExcludedIndexResponse.isAcknowledged(), is(true));
265+
258266
CreateIndexRequest createIndexRequest = new CreateIndexRequest("logs-20200101");
259267
CreateIndexResponse response = highLevelClient().indices().create(createIndexRequest, RequestOptions.DEFAULT);
260268
assertThat(response.isAcknowledged(), is(true));
@@ -264,11 +272,13 @@ public void testAutoFollowing() throws Exception {
264272
CcrStatsResponse ccrStatsResponse = execute(ccrStatsRequest, ccrClient::getCcrStats, ccrClient::getCcrStatsAsync);
265273
assertThat(ccrStatsResponse.getAutoFollowStats().getNumberOfSuccessfulFollowIndices(), equalTo(1L));
266274
assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-20200101"), notNullValue());
275+
assertThat(ccrStatsResponse.getIndicesFollowStats().getShardFollowStats("copy-logs-excluded"), nullValue());
267276
});
268277
assertThat(indexExists("copy-logs-20200101"), is(true));
269278
assertThat(
270279
getIndexSettingsAsMap("copy-logs-20200101"),
271280
hasEntry("index.number_of_replicas", Integer.toString(followerNumberOfReplicas)));
281+
assertThat(indexExists("copy-logs-excluded"), is(false));
272282

273283
GetAutoFollowPatternRequest getAutoFollowPatternRequest =
274284
randomBoolean() ? new GetAutoFollowPatternRequest("pattern1") : new GetAutoFollowPatternRequest();
@@ -279,6 +289,7 @@ public void testAutoFollowing() throws Exception {
279289
assertThat(pattern, notNullValue());
280290
assertThat(pattern.getRemoteCluster(), equalTo(putAutoFollowPatternRequest.getRemoteCluster()));
281291
assertThat(pattern.getLeaderIndexPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexPatterns()));
292+
assertThat(pattern.getLeaderIndexExclusionPatterns(), equalTo(putAutoFollowPatternRequest.getLeaderIndexExclusionPatterns()));
282293
assertThat(pattern.getFollowIndexNamePattern(), equalTo(putAutoFollowPatternRequest.getFollowIndexNamePattern()));
283294
assertThat(pattern.getSettings(), equalTo(autoFollowerPatternSettings));
284295

‎client/rest-high-level/src/test/java/org/elasticsearch/client/CcrRequestConvertersTests.java

+4-1
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,10 @@ public void testForgetFollower() throws IOException {
101101

102102
public void testPutAutofollowPattern() throws Exception {
103103
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest(randomAlphaOfLength(4),
104-
randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
104+
randomAlphaOfLength(4),
105+
Arrays.asList(generateRandomStringArray(4, 4, false)),
106+
Arrays.asList(generateRandomStringArray(4, 4, false))
107+
);
105108
if (randomBoolean()) {
106109
putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
107110
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/GetAutoFollowPatternResponseTests.java

+3
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
3939
for (int i = 0; i < numPatterns; i++) {
4040
String remoteCluster = randomAlphaOfLength(4);
4141
List<String> leaderIndexPatterns = Collections.singletonList(randomAlphaOfLength(4));
42+
List<String> leaderIndexExclusionsPatterns = randomList(0, randomIntBetween(1, 10), () -> randomAlphaOfLength(4));
4243
String followIndexNamePattern = randomAlphaOfLength(4);
4344
final Settings settings =
4445
Settings.builder().put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), randomIntBetween(0, 4)).build();
@@ -89,6 +90,7 @@ protected GetAutoFollowPatternAction.Response createServerTestInstance(XContentT
8990
new AutoFollowMetadata.AutoFollowPattern(
9091
remoteCluster,
9192
leaderIndexPatterns,
93+
leaderIndexExclusionsPatterns,
9294
followIndexNamePattern,
9395
settings,
9496
active,
@@ -124,6 +126,7 @@ protected void assertInstances(GetAutoFollowPatternAction.Response serverTestIns
124126
assertThat(serverPattern.getRemoteCluster(), equalTo(clientPattern.getRemoteCluster()));
125127
assertThat(serverPattern.getLeaderIndexPatterns(), equalTo(clientPattern.getLeaderIndexPatterns()));
126128
assertThat(serverPattern.getFollowIndexPattern(), equalTo(clientPattern.getFollowIndexNamePattern()));
129+
assertThat(serverPattern.getLeaderIndexExclusionPatterns(), equalTo(clientPattern.getLeaderIndexExclusionPatterns()));
127130
assertThat(serverPattern.getSettings(), equalTo(clientPattern.getSettings()));
128131
assertThat(serverPattern.getMaxOutstandingReadRequests(), equalTo(clientPattern.getMaxOutstandingReadRequests()));
129132
assertThat(serverPattern.getMaxOutstandingWriteRequests(), equalTo(clientPattern.getMaxOutstandingWriteRequests()));

‎client/rest-high-level/src/test/java/org/elasticsearch/client/ccr/PutAutoFollowPatternRequestTests.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,10 @@ public class PutAutoFollowPatternRequestTests extends AbstractRequestTestCase<
2828
protected PutAutoFollowPatternRequest createClientTestInstance() {
2929
// Name isn't serialized, because it specified in url path, so no need to randomly generate it here.
3030
PutAutoFollowPatternRequest putAutoFollowPatternRequest = new PutAutoFollowPatternRequest("name",
31-
randomAlphaOfLength(4), Arrays.asList(generateRandomStringArray(4, 4, false)));
31+
randomAlphaOfLength(4),
32+
Arrays.asList(generateRandomStringArray(4, 4, false)),
33+
Arrays.asList(generateRandomStringArray(4, 4, false))
34+
);
3235
if (randomBoolean()) {
3336
putAutoFollowPatternRequest.setFollowIndexNamePattern(randomAlphaOfLength(4));
3437
}
@@ -75,6 +78,7 @@ protected void assertInstances(PutAutoFollowPatternAction.Request serverInstance
7578
assertThat(serverInstance.getName(), equalTo(clientTestInstance.getName()));
7679
assertThat(serverInstance.getRemoteCluster(), equalTo(clientTestInstance.getRemoteCluster()));
7780
assertThat(serverInstance.getLeaderIndexPatterns(), equalTo(clientTestInstance.getLeaderIndexPatterns()));
81+
assertThat(serverInstance.getLeaderIndexExclusionPatterns(), equalTo(clientTestInstance.getLeaderIndexExclusionPatterns()));
7882
assertThat(serverInstance.getFollowIndexNamePattern(), equalTo(clientTestInstance.getFollowIndexNamePattern()));
7983
assertFollowConfig(serverInstance.getParameters(), clientTestInstance);
8084
}

‎client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CCRDocumentationIT.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -474,12 +474,13 @@ public void testPutAutoFollowPattern() throws Exception {
474474
new PutAutoFollowPatternRequest(
475475
"my_pattern", // <1>
476476
"local", // <2>
477-
Arrays.asList("logs-*", "metrics-*") // <3>
477+
Arrays.asList("logs-*", "metrics-*"), // <3>
478+
Arrays.asList("logs-excluded", "metrics-excluded") // <4>
478479
);
479-
request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <4>
480+
request.setFollowIndexNamePattern("copy-{{leader_index}}"); // <5>
480481
Settings settings =
481482
Settings.builder().put("index.number_of_replicas", 0L).build();
482-
request.setSettings(settings); // <5>
483+
request.setSettings(settings); // <6>
483484
// end::ccr-put-auto-follow-pattern-request
484485

485486
// tag::ccr-put-auto-follow-pattern-execute

‎docs/java-rest/high-level/ccr/put_auto_follow_pattern.asciidoc

+3-2
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,9 @@ include-tagged::{doc-tests-file}[{api}-request]
2121
<1> The name of the auto follow pattern.
2222
<2> The name of the remote cluster.
2323
<3> The leader index patterns.
24-
<4> The pattern used to create the follower index
25-
<5> The settings overrides for the follower index
24+
<4> The leader index exclusion patterns.
25+
<5> The pattern used to create the follower index.
26+
<6> The settings overrides for the follower index.
2627

2728
[id="{upid}-{api}-response"]
2829
==== Response

‎docs/reference/ccr/apis/auto-follow/get-auto-follow-pattern.asciidoc

+8
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ PUT /_ccr/auto_follow/my_auto_follow_pattern
2222
[
2323
"leader_index*"
2424
],
25+
"leader_index_exclusion_patterns":
26+
[
27+
"leader_index_001"
28+
],
2529
"follow_index_pattern" : "{{leader_index}}-follower"
2630
}
2731
--------------------------------------------------
@@ -96,6 +100,10 @@ The API returns the following result:
96100
[
97101
"leader_index*"
98102
],
103+
"leader_index_exclusion_patterns":
104+
[
105+
"leader_index_001"
106+
],
99107
"follow_index_pattern" : "{{leader_index}}-follower"
100108
}
101109
}

‎docs/reference/ccr/apis/auto-follow/put-auto-follow-pattern.asciidoc

+14-1
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,18 @@ PUT /_ccr/auto_follow/<auto_follow_pattern_name>
2020
[
2121
"<leader_index_pattern>"
2222
],
23+
"leader_index_exclusion_patterns":
24+
[
25+
"<leader_index_exclusion_pattern>"
26+
],
2327
"follow_index_pattern" : "<follow_index_pattern>"
2428
}
2529
--------------------------------------------------
2630
// TEST[setup:remote_cluster]
2731
// TEST[s/<auto_follow_pattern_name>/auto_follow_pattern_name/]
2832
// TEST[s/<remote_cluster>/remote_cluster/]
2933
// TEST[s/<leader_index_patterns>/leader_index*/]
34+
// TEST[s/<leader_index_exclusion_pattern>//]
3035
// TEST[s/<follow_index_pattern>/{{leader_index}}-follower/]
3136

3237
//////////////////////////
@@ -54,7 +59,10 @@ This API creates a new named collection of
5459
<<ccr-auto-follow,auto-follow patterns>> against the remote cluster
5560
specified in the request body. Newly created indices on the remote cluster
5661
matching any of the specified patterns will be automatically configured as follower
57-
indices.
62+
indices. Additionally, this API can be used to update existing
63+
<<ccr-auto-follow,auto-follow patterns>>. Note that follower indices that were configured automatically
64+
before updating an auto-follow pattern will remain unchanged even if they don't match against
65+
the new patterns.
5866

5967
[[ccr-put-auto-follow-pattern-path-parms]]
6068
==== {api-path-parms-title}
@@ -72,6 +80,11 @@ indices.
7280
(Optional, array) An array of simple index patterns to match against indices
7381
in the remote cluster specified by the `remote_cluster` field.
7482

83+
`leader_index_exclusion_patterns`::
84+
(Optional, array) An array of simple index patterns that can be used to exclude indices
85+
from being auto-followed. Indices in the remote cluster whose names are matching one or
86+
more `leader_index_patterns` and one or more `leader_index_exclusion_patterns` won't be followed.
87+
7588
`follow_index_pattern`::
7689
(Optional, string) The name of follower index. The template `{{leader_index}}`
7790
can be used to derive the name of the follower index from the name of the

0 commit comments

Comments
 (0)