Skip to content

Commit d2bd374

Browse files
DaveCTurnerDaveCTurner
authored
Add GetSnapshotsIT#testAllFeatures (#111786)
The features of get-snapshots API are all tested in isolation or small combinations, but there's no one test which pins down exactly how they all interact. This commit adds such a test, to verify that any future optimization work preserves the observable behaviour. Relates #95345 Relates #104607
1 parent 664573c commit d2bd374

File tree

1 file changed

+255
-0
lines changed

1 file changed

+255
-0
lines changed

‎server/src/internalClusterTest/java/org/elasticsearch/snapshots/GetSnapshotsIT.java

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,24 +10,41 @@
1010

1111
import org.elasticsearch.action.ActionFuture;
1212
import org.elasticsearch.action.ActionRequestValidationException;
13+
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
14+
import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction;
15+
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
1316
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
17+
import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction;
1418
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
1519
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequestBuilder;
1620
import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
1721
import org.elasticsearch.action.admin.cluster.snapshots.get.SnapshotSortKey;
22+
import org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction;
23+
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
24+
import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
25+
import org.elasticsearch.action.support.RefCountingListener;
1826
import org.elasticsearch.cluster.SnapshotsInProgress;
27+
import org.elasticsearch.common.Strings;
1928
import org.elasticsearch.common.settings.Settings;
29+
import org.elasticsearch.core.Predicates;
2030
import org.elasticsearch.repositories.RepositoryMissingException;
31+
import org.elasticsearch.repositories.fs.FsRepository;
2132
import org.elasticsearch.search.sort.SortOrder;
33+
import org.elasticsearch.test.ESTestCase;
34+
import org.elasticsearch.test.hamcrest.ElasticsearchAssertions;
2235
import org.elasticsearch.threadpool.ThreadPool;
2336

2437
import java.nio.file.Path;
2538
import java.util.ArrayList;
2639
import java.util.Collection;
40+
import java.util.Collections;
2741
import java.util.HashSet;
2842
import java.util.List;
2943
import java.util.Map;
44+
import java.util.Objects;
3045
import java.util.Set;
46+
import java.util.function.Predicate;
47+
import java.util.stream.Collectors;
3148

3249
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
3350
import static org.hamcrest.Matchers.empty;
@@ -745,4 +762,242 @@ private static GetSnapshotsRequestBuilder baseGetSnapshotsRequest(String[] repoN
745762
return clusterAdmin().prepareGetSnapshots(TEST_REQUEST_TIMEOUT, repoNames)
746763
.setSnapshots("*", "-" + AbstractSnapshotIntegTestCase.OLD_VERSION_SNAPSHOT_PREFIX + "*");
747764
}
765+
766+
public void testAllFeatures() {
767+
// A test that uses (potentially) as many of the features of the get-snapshots API at once as possible, to verify that they interact
768+
// in the expected order etc.
769+
770+
// Create a few repositories and a few indices
771+
final var repositories = randomList(1, 4, ESTestCase::randomIdentifier);
772+
final var indices = randomList(1, 4, ESTestCase::randomIdentifier);
773+
final var slmPolicies = randomList(1, 4, ESTestCase::randomIdentifier);
774+
775+
safeAwait(l -> {
776+
try (var listeners = new RefCountingListener(l.map(v -> null))) {
777+
for (final var repository : repositories) {
778+
client().execute(
779+
TransportPutRepositoryAction.TYPE,
780+
new PutRepositoryRequest(TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT, repository).type(FsRepository.TYPE)
781+
.settings(Settings.builder().put("location", randomRepoPath()).build()),
782+
listeners.acquire(ElasticsearchAssertions::assertAcked)
783+
);
784+
}
785+
786+
for (final var index : indices) {
787+
client().execute(
788+
TransportCreateIndexAction.TYPE,
789+
new CreateIndexRequest(index, indexSettings(1, 0).build()),
790+
listeners.acquire(ElasticsearchAssertions::assertAcked)
791+
);
792+
}
793+
}
794+
});
795+
ensureGreen();
796+
797+
// Create a few snapshots
798+
final var snapshotInfos = Collections.synchronizedList(new ArrayList<SnapshotInfo>());
799+
safeAwait(l -> {
800+
try (var listeners = new RefCountingListener(l.map(v -> null))) {
801+
for (int i = 0; i < 10; i++) {
802+
client().execute(
803+
TransportCreateSnapshotAction.TYPE,
804+
new CreateSnapshotRequest(
805+
TEST_REQUEST_TIMEOUT,
806+
// at least one snapshot per repository to satisfy consistency checks
807+
i < repositories.size() ? repositories.get(i) : randomFrom(repositories),
808+
randomIdentifier()
809+
).indices(randomNonEmptySubsetOf(indices))
810+
.userMetadata(
811+
randomBoolean() ? Map.of() : Map.of(SnapshotsService.POLICY_ID_METADATA_FIELD, randomFrom(slmPolicies))
812+
)
813+
.waitForCompletion(true),
814+
listeners.acquire(
815+
createSnapshotResponse -> snapshotInfos.add(Objects.requireNonNull(createSnapshotResponse.getSnapshotInfo()))
816+
)
817+
);
818+
}
819+
}
820+
});
821+
822+
Predicate<SnapshotInfo> snapshotInfoPredicate = Predicates.always();
823+
824+
// {repository} path parameter
825+
final String[] requestedRepositories;
826+
if (randomBoolean()) {
827+
requestedRepositories = new String[] { randomFrom("_all", "*") };
828+
} else {
829+
final var selectedRepositories = Set.copyOf(randomNonEmptySubsetOf(repositories));
830+
snapshotInfoPredicate = snapshotInfoPredicate.and(si -> selectedRepositories.contains(si.repository()));
831+
requestedRepositories = selectedRepositories.toArray(new String[0]);
832+
}
833+
834+
// {snapshot} path parameter
835+
final String[] requestedSnapshots;
836+
if (randomBoolean()) {
837+
requestedSnapshots = randomBoolean() ? Strings.EMPTY_ARRAY : new String[] { randomFrom("_all", "*") };
838+
} else {
839+
final var selectedSnapshots = randomNonEmptySubsetOf(snapshotInfos).stream()
840+
.map(si -> si.snapshotId().getName())
841+
.collect(Collectors.toSet());
842+
snapshotInfoPredicate = snapshotInfoPredicate.and(si -> selectedSnapshots.contains(si.snapshotId().getName()));
843+
requestedSnapshots = selectedSnapshots.stream()
844+
// if we have multiple repositories, add a trailing wildcard to each requested snapshot name, because if we specify exact
845+
// names then there must be a snapshot with that name in every requested repository
846+
.map(n -> repositories.size() == 1 && randomBoolean() ? n : n + "*")
847+
.toArray(String[]::new);
848+
}
849+
850+
// ?slm_policy_filter parameter
851+
final String[] requestedSlmPolicies;
852+
switch (between(0, 3)) {
853+
default -> requestedSlmPolicies = Strings.EMPTY_ARRAY;
854+
case 1 -> {
855+
requestedSlmPolicies = new String[] { "*" };
856+
snapshotInfoPredicate = snapshotInfoPredicate.and(
857+
si -> si.userMetadata().get(SnapshotsService.POLICY_ID_METADATA_FIELD) != null
858+
);
859+
}
860+
case 2 -> {
861+
requestedSlmPolicies = new String[] { "_none" };
862+
snapshotInfoPredicate = snapshotInfoPredicate.and(
863+
si -> si.userMetadata().get(SnapshotsService.POLICY_ID_METADATA_FIELD) == null
864+
);
865+
}
866+
case 3 -> {
867+
final var selectedPolicies = Set.copyOf(randomNonEmptySubsetOf(slmPolicies));
868+
requestedSlmPolicies = selectedPolicies.stream()
869+
.map(policy -> randomBoolean() ? policy : policy + "*")
870+
.toArray(String[]::new);
871+
snapshotInfoPredicate = snapshotInfoPredicate.and(
872+
si -> si.userMetadata().get(SnapshotsService.POLICY_ID_METADATA_FIELD) instanceof String policy
873+
&& selectedPolicies.contains(policy)
874+
);
875+
}
876+
}
877+
878+
// ?sort and ?order parameters
879+
final var sortKey = randomFrom(SnapshotSortKey.values());
880+
final var order = randomFrom(SortOrder.values());
881+
// NB we sometimes choose to sort by FAILED_SHARDS, but there are no failed shards in these snapshots. We're still testing the
882+
// fallback sorting by snapshot ID in this case. We also have no multi-shard indices so there's no difference between sorting by
883+
// INDICES and by SHARDS. The actual sorting behaviour for these cases is tested elsewhere, here we're just checking that sorting
884+
// interacts correctly with the other parameters to the API.
885+
886+
// compute the ordered sequence of snapshots which match the repository/snapshot name filters and SLM policy filter
887+
final var selectedSnapshots = snapshotInfos.stream()
888+
.filter(snapshotInfoPredicate)
889+
.sorted(sortKey.getSnapshotInfoComparator(order))
890+
.toList();
891+
892+
final var getSnapshotsRequest = new GetSnapshotsRequest(TEST_REQUEST_TIMEOUT, requestedRepositories, requestedSnapshots).policies(
893+
requestedSlmPolicies
894+
)
895+
// apply sorting params
896+
.sort(sortKey)
897+
.order(order);
898+
899+
// sometimes use ?from_sort_value to skip some items; note that snapshots skipped in this way are subtracted from
900+
// GetSnapshotsResponse.totalCount whereas snapshots skipped by ?after and ?offset are not
901+
final int skippedByFromSortValue;
902+
if (randomBoolean()) {
903+
final var startingSnapshot = randomFrom(snapshotInfos);
904+
getSnapshotsRequest.fromSortValue(switch (sortKey) {
905+
case START_TIME -> Long.toString(startingSnapshot.startTime());
906+
case NAME -> startingSnapshot.snapshotId().getName();
907+
case DURATION -> Long.toString(startingSnapshot.endTime() - startingSnapshot.startTime());
908+
case INDICES, SHARDS -> Integer.toString(startingSnapshot.indices().size());
909+
case FAILED_SHARDS -> "0";
910+
case REPOSITORY -> startingSnapshot.repository();
911+
});
912+
final Predicate<SnapshotInfo> fromSortValuePredicate = snapshotInfo -> {
913+
final var comparison = switch (sortKey) {
914+
case START_TIME -> Long.compare(snapshotInfo.startTime(), startingSnapshot.startTime());
915+
case NAME -> snapshotInfo.snapshotId().getName().compareTo(startingSnapshot.snapshotId().getName());
916+
case DURATION -> Long.compare(
917+
snapshotInfo.endTime() - snapshotInfo.startTime(),
918+
startingSnapshot.endTime() - startingSnapshot.startTime()
919+
);
920+
case INDICES, SHARDS -> Integer.compare(snapshotInfo.indices().size(), startingSnapshot.indices().size());
921+
case FAILED_SHARDS -> 0;
922+
case REPOSITORY -> snapshotInfo.repository().compareTo(startingSnapshot.repository());
923+
};
924+
return order == SortOrder.ASC ? comparison < 0 : comparison > 0;
925+
};
926+
927+
int skipCount = 0;
928+
for (final var snapshotInfo : selectedSnapshots) {
929+
if (fromSortValuePredicate.test(snapshotInfo)) {
930+
skipCount += 1;
931+
} else {
932+
break;
933+
}
934+
}
935+
skippedByFromSortValue = skipCount;
936+
} else {
937+
skippedByFromSortValue = 0;
938+
}
939+
940+
// ?offset parameter
941+
if (randomBoolean()) {
942+
getSnapshotsRequest.offset(between(0, selectedSnapshots.size() + 1));
943+
}
944+
945+
// ?size parameter
946+
if (randomBoolean()) {
947+
getSnapshotsRequest.size(between(1, selectedSnapshots.size() + 1));
948+
}
949+
950+
// compute the expected offset and size of the returned snapshots as indices in selectedSnapshots:
951+
final var expectedOffset = Math.min(selectedSnapshots.size(), skippedByFromSortValue + getSnapshotsRequest.offset());
952+
final var expectedSize = Math.min(
953+
selectedSnapshots.size() - expectedOffset,
954+
getSnapshotsRequest.size() == GetSnapshotsRequest.NO_LIMIT ? Integer.MAX_VALUE : getSnapshotsRequest.size()
955+
);
956+
957+
// get the actual response
958+
final GetSnapshotsResponse getSnapshotsResponse = safeAwait(
959+
l -> client().execute(TransportGetSnapshotsAction.TYPE, getSnapshotsRequest, l)
960+
);
961+
962+
// verify it returns the expected results
963+
assertEquals(
964+
selectedSnapshots.stream().skip(expectedOffset).limit(expectedSize).map(SnapshotInfo::snapshotId).toList(),
965+
getSnapshotsResponse.getSnapshots().stream().map(SnapshotInfo::snapshotId).toList()
966+
);
967+
assertEquals(expectedSize, getSnapshotsResponse.getSnapshots().size());
968+
assertEquals(selectedSnapshots.size() - skippedByFromSortValue, getSnapshotsResponse.totalCount());
969+
assertEquals(selectedSnapshots.size() - expectedOffset - expectedSize, getSnapshotsResponse.remaining());
970+
assertEquals(getSnapshotsResponse.remaining() > 0, getSnapshotsResponse.next() != null);
971+
972+
// now use ?after to page through the rest of the results
973+
var nextRequestAfter = getSnapshotsResponse.next();
974+
var nextExpectedOffset = expectedOffset + expectedSize;
975+
var remaining = getSnapshotsResponse.remaining();
976+
while (nextRequestAfter != null) {
977+
final var nextSize = between(1, remaining);
978+
final var nextRequest = new GetSnapshotsRequest(TEST_REQUEST_TIMEOUT, requestedRepositories, requestedSnapshots)
979+
// same name/policy filters, same ?sort and ?order params, new ?size, but no ?offset or ?from_sort_value because of ?after
980+
.policies(requestedSlmPolicies)
981+
.sort(sortKey)
982+
.order(order)
983+
.size(nextSize)
984+
.after(SnapshotSortKey.decodeAfterQueryParam(nextRequestAfter));
985+
final GetSnapshotsResponse nextResponse = safeAwait(l -> client().execute(TransportGetSnapshotsAction.TYPE, nextRequest, l));
986+
987+
assertEquals(
988+
selectedSnapshots.stream().skip(nextExpectedOffset).limit(nextSize).map(SnapshotInfo::snapshotId).toList(),
989+
nextResponse.getSnapshots().stream().map(SnapshotInfo::snapshotId).toList()
990+
);
991+
assertEquals(nextSize, nextResponse.getSnapshots().size());
992+
assertEquals(selectedSnapshots.size(), nextResponse.totalCount());
993+
assertEquals(remaining - nextSize, nextResponse.remaining());
994+
assertEquals(nextResponse.remaining() > 0, nextResponse.next() != null);
995+
996+
nextRequestAfter = nextResponse.next();
997+
nextExpectedOffset += nextSize;
998+
remaining -= nextSize;
999+
}
1000+
1001+
assertEquals(0, remaining);
1002+
}
7481003
}

0 commit comments

Comments
 (0)