Skip to content

Commit 2a7f73d

Browse files
original-brownbearoriginal-brownbear
authored
Optimize SLM Policy Queries (#79341) (#79446)
Same as #79321 but for SLM policies. Enhances RepositoryData accordingly to enable the optimization.
1 parent 703d6c2 commit 2a7f73d

File tree

8 files changed

+180
-74
lines changed

8 files changed

+180
-74
lines changed

‎plugins/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,8 @@ public void testEnforcedCooldownPeriod() throws IOException {
156156
SnapshotState.SUCCESS,
157157
SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION.minimumCompatibilityVersion(),
158158
0L, // -1 would refresh RepositoryData and find the real version
159-
0L // -1 would refresh RepositoryData and find the real version
159+
0L, // -1 would refresh RepositoryData and find the real version,
160+
"" // null would refresh RepositoryData and find the real version
160161
)));
161162
final BytesReference serialized = BytesReference.bytes(modifiedRepositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(),
162163
SnapshotsService.OLD_SNAPSHOT_FORMAT));

‎server/src/internalClusterTest/java/org/elasticsearch/snapshots/CorruptedBlobStoreRepositoryIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@ public void testHandlingMissingRootLevelSnapshotMetadata() throws Exception {
330330
.collect(
331331
Collectors.toMap(
332332
SnapshotId::getUUID,
333-
s -> new RepositoryData.SnapshotDetails(repositoryData.getSnapshotState(s), null, -1, -1)
333+
s -> new RepositoryData.SnapshotDetails(repositoryData.getSnapshotState(s), null, -1, -1, null)
334334
)
335335
),
336336
Collections.emptyMap(),

‎server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java

Lines changed: 72 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -672,7 +672,7 @@ private static Predicate<SnapshotInfo> buildAfterPredicate(
672672
}
673673
}
674674

675-
private static Predicate<SnapshotInfo> filterBySLMPolicies(String[] slmPolicies) {
675+
private static SnapshotPredicate filterBySLMPolicies(String[] slmPolicies) {
676676
final List<String> includePatterns = new ArrayList<>();
677677
final List<String> excludePatterns = new ArrayList<>();
678678
boolean seenWildcard = false;
@@ -692,25 +692,47 @@ private static Predicate<SnapshotInfo> filterBySLMPolicies(String[] slmPolicies)
692692
final String[] includes = includePatterns.toArray(Strings.EMPTY_ARRAY);
693693
final String[] excludes = excludePatterns.toArray(Strings.EMPTY_ARRAY);
694694
final boolean matchWithoutPolicy = matchNoPolicy;
695-
return snapshotInfo -> {
696-
final Map<String, Object> metadata = snapshotInfo.userMetadata();
697-
final String policy;
698-
if (metadata == null) {
699-
policy = null;
700-
} else {
701-
final Object policyFound = metadata.get(SnapshotsService.POLICY_ID_METADATA_FIELD);
702-
policy = policyFound instanceof String ? (String) policyFound : null;
703-
}
704-
if (policy == null) {
705-
return matchWithoutPolicy;
695+
return new SnapshotPredicate() {
696+
@Override
697+
public boolean matchesPreflight(SnapshotId snapshotId, RepositoryData repositoryData) {
698+
final RepositoryData.SnapshotDetails details = repositoryData.getSnapshotDetails(snapshotId);
699+
final String policy;
700+
if (details == null || (details.getSlmPolicy() == null)) {
701+
// no SLM policy recorded
702+
return true;
703+
} else {
704+
final String policyFound = details.getSlmPolicy();
705+
// empty string means that snapshot was not created by an SLM policy
706+
policy = policyFound.isEmpty() ? null : policyFound;
707+
}
708+
return matchPolicy(includes, excludes, matchWithoutPolicy, policy);
706709
}
707-
if (Regex.simpleMatch(includes, policy) == false) {
708-
return false;
710+
711+
@Override
712+
public boolean matches(SnapshotInfo snapshotInfo) {
713+
final Map<String, Object> metadata = snapshotInfo.userMetadata();
714+
final String policy;
715+
if (metadata == null) {
716+
policy = null;
717+
} else {
718+
final Object policyFound = metadata.get(SnapshotsService.POLICY_ID_METADATA_FIELD);
719+
policy = policyFound instanceof String ? (String) policyFound : null;
720+
}
721+
return matchPolicy(includes, excludes, matchWithoutPolicy, policy);
709722
}
710-
return excludes.length == 0 || Regex.simpleMatch(excludes, policy) == false;
711723
};
712724
}
713725

726+
private static boolean matchPolicy(String[] includes, String[] excludes, boolean matchWithoutPolicy, @Nullable String policy) {
727+
if (policy == null) {
728+
return matchWithoutPolicy;
729+
}
730+
if (Regex.simpleMatch(includes, policy) == false) {
731+
return false;
732+
}
733+
return excludes.length == 0 || Regex.simpleMatch(excludes, policy) == false;
734+
}
735+
714736
private static Predicate<SnapshotInfo> filterByLongOffset(ToLongFunction<SnapshotInfo> extractor, long after, SortOrder order) {
715737
return order == SortOrder.ASC ? info -> after <= extractor.applyAsLong(info) : info -> after >= extractor.applyAsLong(info);
716738
}
@@ -765,19 +787,23 @@ private static final class SnapshotPredicates {
765787
Predicate<SnapshotInfo> snapshotPredicate = null;
766788
final String[] slmPolicies = request.policies();
767789
final String fromSortValue = request.fromSortValue();
790+
BiPredicate<SnapshotId, RepositoryData> preflightPredicate = null;
768791
if (slmPolicies.length > 0) {
769-
snapshotPredicate = filterBySLMPolicies(slmPolicies);
792+
final SnapshotPredicate predicate = filterBySLMPolicies(slmPolicies);
793+
snapshotPredicate = predicate::matches;
794+
preflightPredicate = predicate::matchesPreflight;
770795
}
771796
final GetSnapshotsRequest.SortBy sortBy = request.sort();
772797
final SortOrder order = request.order();
773798
if (fromSortValue == null) {
774-
preflightPredicate = null;
799+
this.preflightPredicate = preflightPredicate;
775800
} else {
776801
final Predicate<SnapshotInfo> fromSortValuePredicate;
802+
final BiPredicate<SnapshotId, RepositoryData> preflightPred;
777803
switch (sortBy) {
778804
case START_TIME:
779805
final long after = Long.parseLong(fromSortValue);
780-
preflightPredicate = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
806+
preflightPred = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
781807
final long startTime = getStartTime(snapshotId, repositoryData);
782808
return startTime == -1 || after <= startTime;
783809
} : (snapshotId, repositoryData) -> {
@@ -787,14 +813,14 @@ private static final class SnapshotPredicates {
787813
fromSortValuePredicate = filterByLongOffset(SnapshotInfo::startTime, after, order);
788814
break;
789815
case NAME:
790-
preflightPredicate = order == SortOrder.ASC
816+
preflightPred = order == SortOrder.ASC
791817
? (snapshotId, repositoryData) -> fromSortValue.compareTo(snapshotId.getName()) <= 0
792818
: (snapshotId, repositoryData) -> fromSortValue.compareTo(snapshotId.getName()) >= 0;
793819
fromSortValuePredicate = null;
794820
break;
795821
case DURATION:
796822
final long afterDuration = Long.parseLong(fromSortValue);
797-
preflightPredicate = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
823+
preflightPred = order == SortOrder.ASC ? (snapshotId, repositoryData) -> {
798824
final long duration = getDuration(snapshotId, repositoryData);
799825
return duration == -1 || afterDuration <= duration;
800826
} : (snapshotId, repositoryData) -> {
@@ -805,22 +831,22 @@ private static final class SnapshotPredicates {
805831
break;
806832
case INDICES:
807833
final int afterIndexCount = Integer.parseInt(fromSortValue);
808-
preflightPredicate = order == SortOrder.ASC
834+
preflightPred = order == SortOrder.ASC
809835
? (snapshotId, repositoryData) -> afterIndexCount <= indexCount(snapshotId, repositoryData)
810836
: (snapshotId, repositoryData) -> afterIndexCount >= indexCount(snapshotId, repositoryData);
811837
fromSortValuePredicate = null;
812838
break;
813839
case REPOSITORY:
814840
// already handled in #maybeFilterRepositories
815-
preflightPredicate = null;
841+
preflightPred = null;
816842
fromSortValuePredicate = null;
817843
break;
818844
case SHARDS:
819-
preflightPredicate = null;
845+
preflightPred = null;
820846
fromSortValuePredicate = filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(fromSortValue), order);
821847
break;
822848
case FAILED_SHARDS:
823-
preflightPredicate = null;
849+
preflightPred = null;
824850
fromSortValuePredicate = filterByLongOffset(SnapshotInfo::failedShards, Integer.parseInt(fromSortValue), order);
825851
break;
826852
default:
@@ -832,6 +858,15 @@ private static final class SnapshotPredicates {
832858
} else if (fromSortValuePredicate != null) {
833859
snapshotPredicate = fromSortValuePredicate.and(snapshotPredicate);
834860
}
861+
if (preflightPredicate == null) {
862+
this.preflightPredicate = preflightPred;
863+
} else {
864+
if (preflightPred != null) {
865+
this.preflightPredicate = preflightPredicate.and(preflightPred);
866+
} else {
867+
this.preflightPredicate = preflightPredicate;
868+
}
869+
}
835870
}
836871
this.snapshotPredicate = snapshotPredicate;
837872
}
@@ -848,6 +883,19 @@ public BiPredicate<SnapshotId, RepositoryData> preflightPredicate() {
848883

849884
}
850885

886+
private interface SnapshotPredicate {
887+
888+
/**
889+
* Checks if a snapshot matches the predicate by testing its {@link SnapshotId} for a given {@link RepositoryData}.
890+
*/
891+
boolean matchesPreflight(SnapshotId snapshotId, RepositoryData repositoryData);
892+
893+
/**
894+
* Checks if a snapshot matches the predicate by testing its {@link SnapshotInfo}.
895+
*/
896+
boolean matches(SnapshotInfo snapshotInfo);
897+
}
898+
851899
private static final class SnapshotsInRepo {
852900

853901
private final List<SnapshotInfo> snapshotInfos;

‎server/src/main/java/org/elasticsearch/repositories/RepositoryData.java

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,8 @@ public boolean hasMissingDetails(SnapshotId snapshotId) {
282282
return snapshotDetails == null
283283
|| snapshotDetails.getVersion() == null
284284
|| snapshotDetails.getStartTimeMillis() == -1
285-
|| snapshotDetails.getEndTimeMillis() == -1;
285+
|| snapshotDetails.getEndTimeMillis() == -1
286+
|| snapshotDetails.getSlmPolicy() == null;
286287
}
287288

288289
/**
@@ -638,6 +639,7 @@ public Map<String, IndexId> resolveNewIndices(List<String> indicesToResolve, Map
638639
private static final String MIN_VERSION = "min_version";
639640
private static final String START_TIME_MILLIS = "start_time_millis";
640641
private static final String END_TIME_MILLIS = "end_time_millis";
642+
private static final String SLM_POLICY = "slm_policy";
641643

642644
/**
643645
* Writes the snapshots metadata and the related indices metadata to x-content.
@@ -724,6 +726,9 @@ public XContentBuilder snapshotsToXContent(final XContentBuilder builder, final
724726
if (snapshotDetails.getEndTimeMillis() != -1) {
725727
builder.field(END_TIME_MILLIS, snapshotDetails.getEndTimeMillis());
726728
}
729+
if (snapshotDetails.getSlmPolicy() != null) {
730+
builder.field(SLM_POLICY, snapshotDetails.getSlmPolicy());
731+
}
727732

728733
builder.endObject();
729734
}
@@ -890,6 +895,7 @@ private static void parseSnapshots(
890895
Version version = null;
891896
long startTimeMillis = -1;
892897
long endTimeMillis = -1;
898+
String slmPolicy = null;
893899
while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
894900
String currentFieldName = parser.currentName();
895901
parser.nextToken();
@@ -917,12 +923,14 @@ private static void parseSnapshots(
917923
assert endTimeMillis == -1;
918924
endTimeMillis = parser.longValue();
919925
break;
926+
case SLM_POLICY:
927+
slmPolicy = stringDeduplicator.computeIfAbsent(parser.text(), Function.identity());
920928
}
921929
}
922930
assert (startTimeMillis == -1) == (endTimeMillis == -1) : "unexpected: " + startTimeMillis + ", " + endTimeMillis + ", ";
923931
final SnapshotId snapshotId = new SnapshotId(name, uuid);
924932
if (state != null || version != null) {
925-
snapshotsDetails.put(uuid, new SnapshotDetails(state, version, startTimeMillis, endTimeMillis));
933+
snapshotsDetails.put(uuid, new SnapshotDetails(state, version, startTimeMillis, endTimeMillis, slmPolicy));
926934
}
927935
snapshots.put(uuid, snapshotId);
928936
if (metaGenerations != null && metaGenerations.isEmpty() == false) {
@@ -1036,7 +1044,7 @@ private static String parseLegacySnapshotUUID(XContentParser parser) throws IOEx
10361044
*/
10371045
public static class SnapshotDetails {
10381046

1039-
public static SnapshotDetails EMPTY = new SnapshotDetails(null, null, -1, -1);
1047+
public static SnapshotDetails EMPTY = new SnapshotDetails(null, null, -1, -1, null);
10401048

10411049
@Nullable // TODO forbid nulls here, this only applies to very old repositories
10421050
private final SnapshotState snapshotState;
@@ -1050,11 +1058,23 @@ public static class SnapshotDetails {
10501058
// May be -1 if unknown, which happens if the snapshot was taken before 7.14 and hasn't been updated yet
10511059
private final long endTimeMillis;
10521060

1053-
public SnapshotDetails(@Nullable SnapshotState snapshotState, @Nullable Version version, long startTimeMillis, long endTimeMillis) {
1061+
// May be null if unknown, which happens if the snapshot was taken before 7.16 and hasn't been updated yet. Empty string indicates
1062+
// that this snapshot was not created by an SLM policy.
1063+
@Nullable
1064+
private final String slmPolicy;
1065+
1066+
public SnapshotDetails(
1067+
@Nullable SnapshotState snapshotState,
1068+
@Nullable Version version,
1069+
long startTimeMillis,
1070+
long endTimeMillis,
1071+
@Nullable String slmPolicy
1072+
) {
10541073
this.snapshotState = snapshotState;
10551074
this.version = version;
10561075
this.startTimeMillis = startTimeMillis;
10571076
this.endTimeMillis = endTimeMillis;
1077+
this.slmPolicy = slmPolicy;
10581078
}
10591079

10601080
@Nullable
@@ -1081,6 +1101,15 @@ public long getEndTimeMillis() {
10811101
return endTimeMillis;
10821102
}
10831103

1104+
/**
1105+
* @return the SLM policy that the snapshot was created by or an empty string if it was not created by an SLM policy or
1106+
* {@code null} if unknown.
1107+
*/
1108+
@Nullable
1109+
public String getSlmPolicy() {
1110+
return slmPolicy;
1111+
}
1112+
10841113
@Override
10851114
public boolean equals(Object o) {
10861115
if (this == o) return true;
@@ -1089,12 +1118,13 @@ public boolean equals(Object o) {
10891118
return startTimeMillis == that.startTimeMillis
10901119
&& endTimeMillis == that.endTimeMillis
10911120
&& snapshotState == that.snapshotState
1092-
&& Objects.equals(version, that.version);
1121+
&& Objects.equals(version, that.version)
1122+
&& Objects.equals(slmPolicy, that.slmPolicy);
10931123
}
10941124

10951125
@Override
10961126
public int hashCode() {
1097-
return Objects.hash(snapshotState, version, startTimeMillis, endTimeMillis);
1127+
return Objects.hash(snapshotState, version, startTimeMillis, endTimeMillis, slmPolicy);
10981128
}
10991129

11001130
}

0 commit comments

Comments
 (0)