From 5ddd2dbe50f25665fdebb7890343d7d32ba72079 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 11 Jan 2021 12:06:33 +0000 Subject: [PATCH 01/43] Introduce repository test kit with speed test Today we rely on blob stores behaving in a certain way so that they can be used correctly as a snapshot repository. There are an increasing number of third-party blob stores that claim to be S3-compatible, but which may not offer a suitably performant implementation of the S3 API. We rely on S3's subtle semantics with concurrent readers and writers, but some blob stores may not implement it correctly. Hitting a corner case in the implementation may be rare in normal use, and may be hard to reproduce or to distinguish from an Elasticsearch bug. This commit introduces a new `POST _repository/.../_speed_test` API which measures the details of the performance of the repository under concurrent load, and exercises the more problematic corners of the API implementation looking for correctness bugs. --- .../apis/repo-speed-test-api.asciidoc | 132 +++ .../apis/snapshot-restore-apis.asciidoc | 1 + .../register-repository.asciidoc | 3 + .../RepositoryVerificationException.java | 6 + .../blobstore/BlobStoreRepository.java | 44 +- .../snapshot-repo-test-kit/build.gradle | 30 + .../snapshot-repo-test-kit/qa/build.gradle | 6 + .../qa/rest/build.gradle | 26 + .../testkit/rest/FsSnapshotRepoTestKitIT.java | 24 + ...pshotRepoTestKitClientYamlTestSuiteIT.java | 23 + .../api/snapshot.repository_speed_test.json | 52 ++ .../rest-api-spec/test/10_speed_test.yml | 57 ++ .../testkit/RepositorySpeedTestIT.java | 320 +++++++ .../testkit/BlobSpeedTestAction.java | 784 ++++++++++++++++++ .../testkit/GetBlobChecksumAction.java | 353 ++++++++ .../blobstore/testkit/RandomBlobContent.java | 82 ++ .../RandomBlobContentBytesReference.java | 86 ++ .../testkit/RandomBlobContentStream.java | 114 +++ .../testkit/RepositorySpeedTestAction.java | 718 ++++++++++++++++ .../testkit/SnapshotRepositoryTestKit.java | 49 ++ .../RestRepositorySpeedTestAction.java | 51 ++ ...stractSnapshotRepoTestKitRestTestCase.java | 36 + .../testkit/RandomBlobContentStreamTests.java | 135 +++ 23 files changed, 3127 insertions(+), 5 deletions(-) create mode 100644 docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc create mode 100644 x-pack/plugin/snapshot-repo-test-kit/build.gradle create mode 100644 x-pack/plugin/snapshot-repo-test-kit/qa/build.gradle create mode 100644 x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle create mode 100644 x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/SnapshotRepoTestKitClientYamlTestSuiteIT.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json create mode 100644 x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRepositorySpeedTestAction.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java diff --git a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc new file mode 100644 index 0000000000000..137aeab151870 --- /dev/null +++ b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc @@ -0,0 +1,132 @@ +[role="xpack"] +[[repo-speed-test-api]] +=== Repository speed test API +++++ +Repository speed test +++++ + +Measures the performance characteristics of a snapshot repository. + +//// +[source,console] +---- +PUT /_snapshot/my_repository +{ + "type": "fs", + "settings": { + "location": "my_backup_location" + } +} +---- +// TESTSETUP +//// + +[source,console] +---- +POST /_snapshot/my_repository/_speed_test?blob_count=10&concurrency=4&max_blob_size=1mb&timeout=120s +---- + +[[repo-speed-test-api-request]] +==== {api-request-title} + +`POST /_snapshot//_speed_test` + +[[repo-speed-test-api-desc]] +==== {api-description-title} + +There are a large number of third-party storage systems available, not all of +which are suitable for use as a snapshot repository by {es}. Some storage +systems perform poorly, or behave incorrectly, especially when accessed +concurrently by multiple clients as the nodes of an {es} cluster do. + +The Repository speed test API performs a collection of read and write +operations on your repository which are specially designed to detect incorrect +behaviour and to measure the performance characteristics of your storage +system. + +Each speed test runs a wide variety of operations generated by a pseudo-random +process. You can seed this process using the optional `seed` parameter in order +to repeat the same set of operations in multiple experiments. Note that the +operations are performed concurrently so may not always happen in the same +order on each run. + +The default values for the parameters to this API are deliberately low to +reduce the impact of running this API accidentally. A realistic experiment +should set `blob_count` to at least `2000` and `max_blob_size` to at least +`2gb`, and will almost certainly need to increase the `timeout` to allow time +for the process to complete successfully. + +If the speed test is successful this API returns details of the testing +process, including how long each operation took. You can use this information +to analyse the performance of your storage system. If any operation fails or +returns an incorrect result, this API returns an error. If the API returns an +error then it may not have removed all the data it wrote to the repository. The +error will indicate the location of any leftover data, and this path is also +recorded in the {es} logs. You should verify yourself that this location has +been cleaned up correctly. If there is still leftover data at the specified +location then you should manually remove it. + +If the connection from your client to {es} is closed while the client is +waiting for the result of the speed test then the test is cancelled. Since a +speed test takes a long time to complete, you may need to configure your client +to wait for longer than usual for a response. On cancellation the speed test +attempts to clean up the data it was writing, but it may not be able to remove +it all. The path to the leftover data is recorded in the {es} logs. You should +verify yourself that this location has been cleaned up correctly. If there is +still leftover data at the specified location then you should manually remove +it. + +NOTE: A speed test writes a substantial amount of data to your repository and +then reads it back again. This consumes bandwidth on the network between the +cluster and the repository, and storage space and IO bandwidth on the +repository itself. You must ensure this load does not affect other users of +these systems. Speed tests respect the repository settings +`max_snapshot_bytes_per_sec` and `max_restore_bytes_per_sec` if available, and +the cluster setting `indices.recovery.max_bytes_per_sec` which you can use to +limit the bandwidth they consume. + +[[repo-speed-test-api-path-params]] +==== {api-path-parms-title} + +``:: +(Required, string) +Name of the snapshot repository to test. + +[[repo-speed-test-api-query-params]] +==== {api-query-parms-title} + +`blob_count`:: +(Optional, integer) The total number of blobs to write to the repository during +the test. Defaults to `100`. For realistic experiments you should set this to +at least `2000`. + +`concurrency`:: +(Optional, integer) The number of write operations to perform concurrently. +Defaults to `10`. + +`seed`:: +(Optional, integer) The seed for the pseudo-random number generator used to +generate the list of operations performed during the test. To repeat the same +set of operations in multiple experiments, use the same seed in each +experiment. + +`max_blob_size`:: +(Optional, <>) The maximum size of a blob to be written +during the test. Defaults to `10mb`. For realistic experiments you should set +this to at least `2gb`. + +`master_timeout`:: +(Optional, <>) Specifies the period of time to wait for +a connection to the master node. If no response is received before the timeout +expires, the request fails and returns an error. Defaults to `30s`. + +`timeout`:: +(Optional, <>) Specifies the period of time to wait for +the test to complete. If no response is received before the timeout expires, +the test is cancelled and returns an error. Defaults to `30s`. + +[role="child_attributes"] +[[repo-speed-test-api-response-body]] +==== {api-response-body-title} + +TODO diff --git a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc index cf70d3bcb2eab..8ffd1834b3b73 100644 --- a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc +++ b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc @@ -27,6 +27,7 @@ For more information, see <>. include::put-repo-api.asciidoc[] include::verify-repo-api.asciidoc[] +include::repo-speed-test-api.asciidoc[] include::get-repo-api.asciidoc[] include::delete-repo-api.asciidoc[] include::clean-up-repo-api.asciidoc[] diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc index 1f210326d225a..96ab4d73e2151 100644 --- a/docs/reference/snapshot-restore/register-repository.asciidoc +++ b/docs/reference/snapshot-restore/register-repository.asciidoc @@ -258,6 +258,9 @@ POST /_snapshot/my_unverified_backup/_verify It returns a list of nodes where repository was successfully verified or an error message if verification process failed. +If desired, you can also test a repository more thoroughly using the +<>. + [discrete] [[snapshots-repository-cleanup]] === Repository cleanup diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoryVerificationException.java b/server/src/main/java/org/elasticsearch/repositories/RepositoryVerificationException.java index 2574ff3b09266..2fe7dbca38aa1 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoryVerificationException.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoryVerificationException.java @@ -46,5 +46,11 @@ public RestStatus status() { public RepositoryVerificationException(StreamInput in) throws IOException{ super(in); } + + @Override + public synchronized Throwable fillInStackTrace() { + // stack trace for a verification failure is uninteresting, the message has all the information we need + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index d53eabc8bee88..85b283168c9f3 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -2289,17 +2289,47 @@ private static ActionListener fileQueueListener(BlockingQueue rateLimiterSupplier, CounterMetric metric) { - return new RateLimitingInputStream(stream, rateLimiterSupplier, metric::inc); + private static InputStream maybeRateLimit( + InputStream stream, + Supplier rateLimiterSupplier, + RateLimitingInputStream.Listener throttleListener) { + return new RateLimitingInputStream(stream, rateLimiterSupplier, throttleListener); } + /** + * Wrap the restore rate limiter (controlled by the repository setting `max_restore_bytes_per_sec` and the cluster setting + * `indices.recovery.max_bytes_per_sec`) around the given stream. Any throttling is reported to the given listener and not otherwise + * recorded in the value returned by {@link BlobStoreRepository#getRestoreThrottleTimeInNanos}. + */ public InputStream maybeRateLimitRestores(InputStream stream) { - return maybeRateLimit(maybeRateLimit(stream, () -> restoreRateLimiter, restoreRateLimitingTimeInNanos), - recoverySettings::rateLimiter, restoreRateLimitingTimeInNanos); + return maybeRateLimitRestores(stream, restoreRateLimitingTimeInNanos::inc); } + /** + * Wrap the restore rate limiter (controlled by the repository setting `max_restore_bytes_per_sec` and the cluster setting + * `indices.recovery.max_bytes_per_sec`) around the given stream. Any throttling is recorded in the value returned by {@link + * BlobStoreRepository#getRestoreThrottleTimeInNanos}. + */ + public InputStream maybeRateLimitRestores(InputStream stream, RateLimitingInputStream.Listener throttleListener) { + return maybeRateLimit(maybeRateLimit(stream, () -> restoreRateLimiter, throttleListener), + recoverySettings::rateLimiter, throttleListener); + } + + /** + * Wrap the snapshot rate limiter (controlled by the repository setting `max_snapshot_bytes_per_sec`) around the given stream. Any + * throttling is recorded in the value returned by {@link BlobStoreRepository#getSnapshotThrottleTimeInNanos()}. + */ public InputStream maybeRateLimitSnapshots(InputStream stream) { - return maybeRateLimit(stream, () -> snapshotRateLimiter, snapshotRateLimitingTimeInNanos); + return maybeRateLimitSnapshots(stream, snapshotRateLimitingTimeInNanos::inc); + } + + /** + * Wrap the snapshot rate limiter (controlled by the repository setting `max_snapshot_bytes_per_sec`) around the given stream. Any + * throttling is reported to the given listener and not otherwise recorded in the value returned by {@link + * BlobStoreRepository#getSnapshotThrottleTimeInNanos()}. + */ + public InputStream maybeRateLimitSnapshots(InputStream stream, RateLimitingInputStream.Listener throttleListener) { + return maybeRateLimit(stream, () -> snapshotRateLimiter, throttleListener); } @Override @@ -2539,6 +2569,10 @@ private static void failStoreIfCorrupted(Store store, Exception e) { } } + public boolean supportURLRepo() { + return supportURLRepo; + } + /** * The result of removing a snapshot from a shard folder in the repository. */ diff --git a/x-pack/plugin/snapshot-repo-test-kit/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/build.gradle new file mode 100644 index 0000000000000..96eb6b3463ac9 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/build.gradle @@ -0,0 +1,30 @@ +apply plugin: 'elasticsearch.internal-cluster-test' +apply plugin: 'elasticsearch.esplugin' +esplugin { + name 'snapshot-repo-test-kit' + description 'A plugin for a test kit for snapshot repositories' + classname 'org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-snapshot-repo-test-kit' + +dependencies { + compileOnly project(path: xpackModule('core'), configuration: 'default') + internalClusterTestImplementation project(path: xpackModule('core'), configuration: 'testArtifacts') +} + +addQaCheckDependencies() + +configurations { + testArtifacts.extendsFrom testRuntime + testArtifacts.extendsFrom testImplementation +} + +def testJar = tasks.register("testJar", Jar) { + appendix 'test' + from sourceSets.test.output +} + +artifacts { + testArtifacts testJar +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/build.gradle new file mode 100644 index 0000000000000..72cd4bba91169 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/build.gradle @@ -0,0 +1,6 @@ +apply plugin: 'elasticsearch.build' +tasks.named("test").configure { enabled = false } + +dependencies { + api project(':test:framework') +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle new file mode 100644 index 0000000000000..1fdadcfa0b1c0 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle @@ -0,0 +1,26 @@ +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.rest-resources' + +dependencies { + testImplementation project(path: xpackModule('snapshot-repo-test-kit'), configuration: 'testArtifacts') +} + +final File repoDir = file("$buildDir/testclusters/repo") + +tasks.named("integTest").configure { + systemProperty 'tests.path.repo', repoDir +} + +testClusters.matching { it.name == "integTest" }.configureEach { + testDistribution = 'DEFAULT' + setting 'path.repo', repoDir.absolutePath +} + +restResources { + restApi { + includeCore 'indices', 'search', 'bulk', 'snapshot', 'nodes', '_common' + includeXpack 'snapshot_repo_test_kit' + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java new file mode 100644 index 0000000000000..e56a4e2de4d37 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit.rest; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.repositories.blobstore.testkit.AbstractSnapshotRepoTestKitRestTestCase; +import org.elasticsearch.repositories.fs.FsRepository; + +public class FsSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestTestCase { + + @Override + protected String repositoryType() { + return FsRepository.TYPE; + } + + @Override + protected Settings repositorySettings() { + return Settings.builder().put("location", System.getProperty("tests.path.repo")).build(); + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/SnapshotRepoTestKitClientYamlTestSuiteIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/SnapshotRepoTestKitClientYamlTestSuiteIT.java new file mode 100644 index 0000000000000..bb20d36c73181 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/SnapshotRepoTestKitClientYamlTestSuiteIT.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit.rest; + +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +public class SnapshotRepoTestKitClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public SnapshotRepoTestKitClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json new file mode 100644 index 0000000000000..37fb28760a971 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json @@ -0,0 +1,52 @@ +{ + "snapshot.repository_speed_test":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html", + "description":"Performs a speed test on a repository" + }, + "stability":"stable", + "visibility":"public", + "url":{ + "paths":[ + { + "path":"/_snapshot/{repository}/_speed_test", + "methods":[ + "POST" + ], + "parts":{ + "repository":{ + "type":"string", + "description":"A repository name" + } + } + } + ] + }, + "params":{ + "blob_count":{ + "type":"number", + "description":"Number of blobs to create during the test" + }, + "concurrency":{ + "type":"number", + "description":"Number of operations to run concurrently during the test" + }, + "seed":{ + "type":"number", + "description":"Seed for the random number generator used to create the test workload" + }, + "max_blob_size":{ + "type":"string", + "description":"Maximum size of a blob to create during the test, e.g '1gb' or '100mb'" + }, + "timeout":{ + "type":"time", + "description":"Explicit operation timeout" + }, + "master_timeout":{ + "type":"time", + "description":"Explicit operation timeout for connection to master node" + } + } + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml new file mode 100644 index 0000000000000..0c688dd2abf85 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml @@ -0,0 +1,57 @@ +--- +setup: + + - do: + snapshot.create_repository: + repository: test_repo + body: + type: fs + settings: + location: "test_repo_loc" + + - do: + snapshot.create_repository: + repository: test_repo_readonly + body: + type: fs + settings: + readonly: true + location: "test_repo_loc" + +--- +"Speed test fails on readonly repositories": + - skip: + version: "- 7.99.99" + reason: "introduced in 8.0" + + - do: + catch: bad_request + snapshot.repository_speed_test: + repository: test_repo_readonly + + - match: { status: 400 } + - match: { error.type: illegal_argument_exception } + - match: { error.reason: "repository [test_repo_readonly] is read-only" } + +--- +"Speed test": + - skip: + version: "- 7.99.99" + reason: "introduced in 8.0" + + - do: + snapshot.repository_speed_test: + repository: test_repo + blob_count: 10 + concurrency: 5 + max_blob_size: 1mb + + - match: { repository: test_repo } + - match: { blob_count: 10 } + - match: { concurrency: 5 } + - match: { max_blob_size: 1mb } + - is_true: seed + - is_true: blob_path + - is_true: blobs + - gte: { listing_nanos: 0} + - gte: { delete_nanos: 0} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java new file mode 100644 index 0000000000000..0d5a7c9db34d4 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java @@ -0,0 +1,320 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobMetadata; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.DeleteResult; +import org.elasticsearch.common.blobstore.support.PlainBlobMetadata; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.RepositoryPlugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.junit.Before; + +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.startsWith; + +public class RepositorySpeedTestIT extends AbstractSnapshotIntegTestCase { + + @Before + public void suppressConsistencyChecks() { + disableRepoConsistencyCheck("repository is not used for snapshots"); + } + + @Override + protected Collection> nodePlugins() { + return List.of(TestPlugin.class, LocalStateCompositeXPackPlugin.class, SnapshotRepositoryTestKit.class); + } + + public void testFoo() { + + createRepository("test-repo", TestPlugin.DISRUPTABLE_REPO_TYPE); + + final DisruptableBlobStore blobStore = new DisruptableBlobStore(); + for (final RepositoriesService repositoriesService : internalCluster().getInstances(RepositoriesService.class)) { + try { + ((DisruptableRepository) repositoriesService.repository("test-repo")).setBlobStore(blobStore); + } catch (RepositoryMissingException e) { + // it's only present on voting masters and data nodes + } + } + + final RepositorySpeedTestAction.Request request = new RepositorySpeedTestAction.Request("test-repo"); + + if (randomBoolean()) { + request.concurrency(between(1, 5)); + blobStore.ensureMaxWriteConcurrency(request.getConcurrency()); + } + + if (randomBoolean()) { + request.blobCount(between(1, 100)); + blobStore.ensureMaxBlobCount(request.getBlobCount()); + } + + if (request.getBlobCount() > 3 || randomBoolean()) { + // only use the default blob size of 10MB if writing a small number of blobs, since this is all in-memory + request.maxBlobSize(new ByteSizeValue(between(1, 2048))); + blobStore.ensureMaxBlobSize(request.getMaxBlobSize().getBytes()); + } + + request.timeout(TimeValue.timeValueSeconds(5)); + + final RepositorySpeedTestAction.Response response = client().execute(RepositorySpeedTestAction.INSTANCE, request).actionGet(); + + assertThat(response.status(), equalTo(RestStatus.OK)); + } + + public static class TestPlugin extends Plugin implements RepositoryPlugin { + + static final String DISRUPTABLE_REPO_TYPE = "disruptable"; + + @Override + public Map getRepositories( + Environment env, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + BigArrays bigArrays, + RecoverySettings recoverySettings + ) { + return Map.of( + DISRUPTABLE_REPO_TYPE, + metadata -> new DisruptableRepository( + metadata, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings, + new BlobPath() + ) + ); + } + } + + static class DisruptableRepository extends BlobStoreRepository { + + private final AtomicReference blobStoreRef = new AtomicReference<>(); + + DisruptableRepository( + RepositoryMetadata metadata, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + BigArrays bigArrays, + RecoverySettings recoverySettings, + BlobPath basePath + ) { + super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); + } + + void setBlobStore(BlobStore blobStore) { + assertTrue(blobStoreRef.compareAndSet(null, blobStore)); + } + + @Override + protected BlobStore createBlobStore() { + final BlobStore blobStore = blobStoreRef.get(); + assertNotNull(blobStore); + return blobStore; + } + + @Override + public String startVerification() { + return null; // suppress verify-on-register since we need to wire up all the nodes' repositories first + } + } + + static class DisruptableBlobStore implements BlobStore { + + private final Map blobContainers = ConcurrentCollections.newConcurrentMap(); + private Semaphore writeSemaphore = new Semaphore(new RepositorySpeedTestAction.Request("dummy").getConcurrency()); + private int maxBlobCount = new RepositorySpeedTestAction.Request("dummy").getBlobCount(); + private long maxBlobSize = new RepositorySpeedTestAction.Request("dummy").getMaxBlobSize().getBytes(); + + @Override + public BlobContainer blobContainer(BlobPath path) { + assertThat(path.buildAsString(), startsWith("temp-speed-test-")); + return blobContainers.computeIfAbsent( + path.buildAsString(), + ignored -> new DisruptableBlobContainer(path, this::deleteContainer, writeSemaphore, maxBlobCount, maxBlobSize) + ); + } + + private void deleteContainer(DisruptableBlobContainer container) { + assertNotNull("container " + container.path() + " not found", blobContainers.remove(container.path().buildAsString())); + } + + @Override + public void close() {} + + public void ensureMaxWriteConcurrency(int concurrency) { + writeSemaphore = new Semaphore(concurrency); + } + + public void ensureMaxBlobCount(int maxBlobCount) { + this.maxBlobCount = maxBlobCount; + } + + public void ensureMaxBlobSize(long maxBlobSize) { + this.maxBlobSize = maxBlobSize; + } + } + + static class DisruptableBlobContainer implements BlobContainer { + + private static final byte[] EMPTY = new byte[0]; + + private final BlobPath path; + private final Consumer deleteContainer; + private final Semaphore writeSemaphore; + private final int maxBlobCount; + private final long maxBlobSize; + private final Map blobs = ConcurrentCollections.newConcurrentMap(); + + DisruptableBlobContainer( + BlobPath path, + Consumer deleteContainer, + Semaphore writeSemaphore, + int maxBlobCount, + long maxBlobSize + ) { + this.path = path; + this.deleteContainer = deleteContainer; + this.writeSemaphore = writeSemaphore; + this.maxBlobCount = maxBlobCount; + this.maxBlobSize = maxBlobSize; + } + + @Override + public BlobPath path() { + return path; + } + + @Override + public boolean blobExists(String blobName) { + return blobs.containsKey(blobName); + } + + @Override + public InputStream readBlob(String blobName) throws IOException { + final byte[] contents = blobs.get(blobName); + if (contents == null) { + throw new FileNotFoundException(blobName + " not found"); + } + return new ByteArrayInputStream(contents); + } + + @Override + public InputStream readBlob(String blobName, long position, long length) throws IOException { + final byte[] contents = blobs.get(blobName); + if (contents == null) { + throw new FileNotFoundException(blobName + " not found"); + } + final int truncatedLength = Math.toIntExact(Math.min(length, contents.length - position)); + return new ByteArrayInputStream(contents, Math.toIntExact(position), truncatedLength); + } + + @Override + public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { + assertTrue("must only write blob [" + blobName + "] non-atomically if it doesn't already exist", failIfAlreadyExists); + assertNull("blob [" + blobName + "] must not exist", blobs.get(blobName)); + + blobs.put(blobName, EMPTY); + writeBlobAtomic(blobName, inputStream, blobSize, false); + } + + @Override + public void writeBlob(String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + writeBlob(blobName, bytes.streamInput(), bytes.length(), failIfAlreadyExists); + } + + @Override + public void writeBlobAtomic(String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + writeBlobAtomic(blobName, bytes.streamInput(), bytes.length(), failIfAlreadyExists); + } + + private void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) + throws IOException { + + if (failIfAlreadyExists) { + assertNull("blob [" + blobName + "] must not exist", blobs.get(blobName)); + } + + assertThat(blobSize, lessThanOrEqualTo(maxBlobSize)); + + assertTrue(writeSemaphore.tryAcquire()); + try { + final byte[] contents = inputStream.readAllBytes(); + assertThat((long) contents.length, equalTo(blobSize)); + blobs.put(blobName, contents); + assertThat(blobs.size(), lessThanOrEqualTo(maxBlobCount)); + } finally { + writeSemaphore.release(); + } + } + + @Override + public DeleteResult delete() { + deleteContainer.accept(this); + return new DeleteResult(blobs.size(), blobs.values().stream().mapToLong(b -> b.length).sum()); + } + + @Override + public void deleteBlobsIgnoringIfNotExists(List blobNames) { + blobs.keySet().removeAll(blobNames); + } + + @Override + public Map listBlobs() { + return blobs.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new PlainBlobMetadata(e.getKey(), e.getValue().length))); + } + + @Override + public Map children() { + return Map.of(); + } + + @Override + public Map listBlobsByPrefix(String blobNamePrefix) { + final Map blobMetadataByName = listBlobs(); + blobMetadataByName.keySet().removeIf(s -> s.startsWith(blobNamePrefix) == false); + return blobMetadataByName; + } + } + +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java new file mode 100644 index 0000000000000..8b6a5d1743ee4 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java @@ -0,0 +1,784 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.StepListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.ThreadedActionListener; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.InputStreamStreamInput; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.RepositoryVerificationException; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskAwareRequest; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongPredicate; +import java.util.stream.Collectors; + +/** + * Action which instructs a node to write a blob to the blob store and verify that it can be read correctly by other nodes. The other nodes + * may read the whole blob or just a range; we verify the data that is read by checksum using {@link GetBlobChecksumAction}. + * + * The other nodes may optionally be instructed to attempt their read just before the write completes (which may indicate that the blob is + * not found but must not yield partial data), and may optionally overwrite the blob while the reads are ongoing (which may yield either + * version of the blob, but again must not yield partial data). Usually, however, we write once and only read after the write completes, and + * in this case we insist that the read succeeds. + * + * On success, details of how long everything took are returned. On failure, cancels the remote read tasks to try and avoid consuming + * unnecessary resources. + */ +public class BlobSpeedTestAction extends ActionType { + + private static final Logger logger = LogManager.getLogger(BlobSpeedTestAction.class); + + public static final BlobSpeedTestAction INSTANCE = new BlobSpeedTestAction(); + public static final String NAME = "cluster:admin/repository/speed_test/blob"; + + private BlobSpeedTestAction() { + super(NAME, Response::new); + } + + public static class TransportAction extends HandledTransportAction { + + private static final Logger logger = BlobSpeedTestAction.logger; + + private final RepositoriesService repositoriesService; + private final TransportService transportService; + + @Inject + public TransportAction(TransportService transportService, ActionFilters actionFilters, RepositoriesService repositoriesService) { + super(NAME, transportService, actionFilters, Request::new, ThreadPool.Names.SNAPSHOT); + this.repositoriesService = repositoriesService; + this.transportService = transportService; + } + + @Override + protected void doExecute(Task task, Request request, ActionListener listener) { + final Repository repository = repositoriesService.repository(request.getRepositoryName()); + if (repository instanceof BlobStoreRepository == false) { + throw new IllegalArgumentException("repository [" + request.getRepositoryName() + "] is not a blob-store repository"); + } + if (repository.isReadOnly()) { + throw new IllegalArgumentException("repository [" + request.getRepositoryName() + "] is read-only"); + } + final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository; + final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(new BlobPath().add(request.blobPath)); + + logger.trace("handling [{}]", request); + + assert task instanceof CancellableTask; + new BlobSpeedTest(transportService, (CancellableTask) task, request, blobStoreRepository, blobContainer, listener).run(); + } + } + + /** + * Speed test of a single blob, performing the write(s) and orchestrating the read(s). + */ + static class BlobSpeedTest { + private final TransportService transportService; + private final CancellableTask task; + private final BlobSpeedTestAction.Request request; + private final BlobStoreRepository repository; + private final BlobContainer blobContainer; + private final ActionListener listener; + private final Random random; + private final boolean checksumWholeBlob; + private final long checksumStart; + private final long checksumEnd; + private final List earlyReadNodes; + private final List readNodes; + private final GroupedActionListener readNodesListener; + private final StepListener write1Step = new StepListener<>(); + private final StepListener write2Step = new StepListener<>(); + + BlobSpeedTest( + TransportService transportService, + CancellableTask task, + BlobSpeedTestAction.Request request, + BlobStoreRepository repository, + BlobContainer blobContainer, + ActionListener listener + ) { + this.transportService = transportService; + this.task = task; + this.request = request; + this.repository = repository; + this.blobContainer = blobContainer; + this.listener = listener; + this.random = new Random(this.request.seed); + + checksumWholeBlob = random.nextBoolean(); + if (checksumWholeBlob) { + checksumStart = 0L; + checksumEnd = this.request.targetLength; + } else { + checksumStart = randomLongBetween(0L, this.request.targetLength); + checksumEnd = randomLongBetween(checksumStart + 1, this.request.targetLength + 1); + } + + final ArrayList nodes = new ArrayList<>(request.nodes); // copy for shuffling purposes + if (request.readEarly) { + Collections.shuffle(nodes, random); + earlyReadNodes = nodes.stream().limit(2).collect(Collectors.toList()); // TODO magic 2 + } else { + earlyReadNodes = List.of(); + } + Collections.shuffle(nodes, random); + readNodes = nodes.stream().limit(10).collect(Collectors.toList()); // TODO magic 10 + + final StepListener> readsCompleteStep = new StepListener<>(); + readNodesListener = new GroupedActionListener<>( + new ThreadedActionListener<>(logger, transportService.getThreadPool(), ThreadPool.Names.SNAPSHOT, readsCompleteStep, false), + earlyReadNodes.size() + readNodes.size() + ); + + // The order is important in this chain: if writing fails then we may never even start all the reads, and we want to cancel + // any read tasks that were started, but the reads step only fails after all the reads have completed so there's no need to + // cancel anything. + write1Step.whenComplete( + write1Details -> write2Step.whenComplete( + write2Details -> readsCompleteStep.whenComplete( + responses -> onReadsComplete(responses, write1Details, write2Details), + this::cleanUpAndReturnFailure + ), + this::cancelReadsCleanUpAndReturnFailure + ), + this::cancelReadsCleanUpAndReturnFailure + ); + } + + void run() { + writeRandomBlob(request.readEarly || random.nextBoolean(), true, this::doReadBeforeWriteComplete, write1Step); + + if (request.writeAndOverwrite) { + assert request.targetLength <= Integer.MAX_VALUE : "oversized atomic write"; + write1Step.whenComplete(ignored -> writeRandomBlob(true, false, this::doReadAfterWrite, write2Step), ignored -> {}); + } else { + write2Step.onResponse(null); + doReadAfterWrite(); + } + } + + private void writeRandomBlob(boolean atomic, boolean failIfExists, Runnable onLastRead, StepListener stepListener) { + assert atomic == false || request.targetLength <= Integer.MAX_VALUE : "oversized atomic write"; + final RandomBlobContent content = new RandomBlobContent( + request.getRepositoryName(), + random.nextLong(), + task::isCancelled, + onLastRead + ); + final AtomicLong throttledNanos = new AtomicLong(); + + logger.trace("writing blob [atomic={}, failIfExists={}] for [{}]", atomic, failIfExists, request.getDescription()); + final long startNanos = System.nanoTime(); + ActionListener.completeWith(stepListener, () -> { + if (atomic || (request.targetLength <= Integer.MAX_VALUE && random.nextBoolean())) { + final RandomBlobContentBytesReference bytesReference = new RandomBlobContentBytesReference( + content, + Math.toIntExact(request.getTargetLength()) + ) { + @Override + public StreamInput streamInput() throws IOException { + return new InputStreamStreamInput( + repository.maybeRateLimitSnapshots(super.streamInput(), throttledNanos::addAndGet) + ); + } + }; + if (atomic) { + blobContainer.writeBlobAtomic(request.blobName, bytesReference, failIfExists); + } else { + blobContainer.writeBlob(request.blobName, bytesReference, failIfExists); + } + } else { + blobContainer.writeBlob( + request.blobName, + repository.maybeRateLimitSnapshots( + new RandomBlobContentStream(content, request.getTargetLength()), + throttledNanos::addAndGet + ), + request.targetLength, + failIfExists + ); + } + final long elapsedNanos = System.nanoTime() - startNanos; + final long checksum = content.getChecksum(checksumStart, checksumEnd); + logger.trace("finished writing blob for [{}], got checksum [{}]", request.getDescription(), checksum); + return new WriteDetails(request.targetLength, elapsedNanos, throttledNanos.get(), checksum); + }); + } + + private void doReadBeforeWriteComplete() { + if (earlyReadNodes.isEmpty() == false) { + logger.trace("sending read request to [{}] for [{}] before write complete", earlyReadNodes, request.getDescription()); + readOnNodes(earlyReadNodes, true); + } + } + + private void doReadAfterWrite() { + logger.trace("sending read request to [{}] for [{}] after write complete", readNodes, request.getDescription()); + readOnNodes(readNodes, false); + } + + private void readOnNodes(List nodes, boolean beforeWriteComplete) { + for (DiscoveryNode node : nodes) { + if (task.isCancelled()) { + // record dummy response since we're already on the path to failure + readNodesListener.onResponse( + new NodeResponse(node, beforeWriteComplete, GetBlobChecksumAction.Response.BLOB_NOT_FOUND) + ); + } else { + // no need for extra synchronization after checking if we were cancelled a couple of lines ago -- we haven't notified + // the outer listener yet so any bans on the children are still in place + transportService.sendChildRequest( + node, + GetBlobChecksumAction.NAME, + getBlobChecksumRequest(), + task, + TransportRequestOptions.EMPTY, + new ActionListenerResponseHandler<>( + readNodesListener.map(r -> makeNodeResponse(node, beforeWriteComplete, r)), + GetBlobChecksumAction.Response::new + ) + ); + } + } + } + + private GetBlobChecksumAction.Request getBlobChecksumRequest() { + return new GetBlobChecksumAction.Request( + request.getRepositoryName(), + request.getBlobPath(), + request.getBlobName(), + checksumStart, + checksumWholeBlob ? 0L : checksumEnd + ); + } + + private NodeResponse makeNodeResponse(DiscoveryNode node, boolean beforeWriteComplete, GetBlobChecksumAction.Response response) { + logger.trace( + "received read response [{}] from [{}] for [{}] [beforeWriteComplete={}]", + response, + node, + request.getDescription(), + beforeWriteComplete + ); + return new NodeResponse(node, beforeWriteComplete, response); + } + + private void cancelReadsCleanUpAndReturnFailure(Exception exception) { + transportService.getTaskManager().cancelTaskAndDescendants(task, "task failed", false, ActionListener.wrap(() -> {})); + cleanUpAndReturnFailure(exception); + } + + private void cleanUpAndReturnFailure(Exception exception) { + logger.trace(new ParameterizedMessage("speed test failed [{}] cleaning up", request.getDescription()), exception); + try { + blobContainer.deleteBlobsIgnoringIfNotExists(List.of(request.blobName)); + } catch (IOException ioException) { + exception.addSuppressed(ioException); + logger.warn( + new ParameterizedMessage( + "failure during post-failure cleanup while speed-testing repository [{}], you may need to manually remove [{}/{}]", + request.getRepositoryName(), + request.getBlobPath(), + request.getBlobName() + ), + exception + ); + } + listener.onFailure(exception); + } + + private void onReadsComplete(Collection responses, WriteDetails write1Details, @Nullable WriteDetails write2Details) { + if (task.isCancelled()) { + cleanUpAndReturnFailure( + new RepositoryVerificationException( + request.getRepositoryName(), + "cancelled [" + request.getDescription() + "] during checksum verification" + ) + ); + return; + } + + final long checksumLength = checksumEnd - checksumStart; + final String expectedChecksumDescription; + final LongPredicate checksumPredicate; + if (write2Details == null) { + checksumPredicate = l -> l == write1Details.checksum; + expectedChecksumDescription = Long.toString(write1Details.checksum); + } else { + checksumPredicate = l -> l == write1Details.checksum || l == write2Details.checksum; + expectedChecksumDescription = write1Details.checksum + " or " + write2Details.checksum; + } + + RepositoryVerificationException failure = null; + for (final NodeResponse nodeResponse : responses) { + final GetBlobChecksumAction.Response response = nodeResponse.response; + final RepositoryVerificationException nodeFailure; + if (response.isNotFound()) { + if (request.readEarly) { + nodeFailure = null; // "not found" is legitimate iff we tried to read it before the write completed + } else if (request.writeAndOverwrite) { + nodeFailure = null; // overwrites surprisingly not necessarily atomic; TODO are we ok with this?? + } else { + nodeFailure = new RepositoryVerificationException( + request.getRepositoryName(), + "node [" + + nodeResponse.node + + "] reported blob not found after it was written [" + + request.getDescription() + + "]" + ); + } + } else { + final long actualChecksum = response.getChecksum(); + if (response.getBytesRead() == checksumLength && checksumPredicate.test(actualChecksum)) { + nodeFailure = null; // checksum ok + } else { + nodeFailure = new RepositoryVerificationException( + request.getRepositoryName(), + "node [" + + nodeResponse.node + + "] failed during speed test: expected to read [" + + checksumStart + + "-" + + checksumEnd + + "], [" + + checksumLength + + "] bytes, with checksum [" + + expectedChecksumDescription + + "] but read [" + + response + + "]" + ); + } + } + + if (nodeFailure != null) { + if (failure == null) { + failure = nodeFailure; + } else { + failure.addSuppressed(nodeFailure); + } + } + } + if (failure != null) { + cleanUpAndReturnFailure(failure); + return; + } + + final long overwriteElapsedNanos = write2Details == null ? 0L : write2Details.elapsedNanos; + final long overwriteThrottledNanos = write2Details == null ? 0L : write2Details.throttledNanos; + listener.onResponse( + new Response( + transportService.getLocalNode().getId(), + transportService.getLocalNode().getName(), + request.blobName, + request.targetLength, + request.readEarly, + request.writeAndOverwrite, + checksumStart, + checksumEnd, + write1Details.elapsedNanos, + overwriteElapsedNanos, + write1Details.throttledNanos + overwriteThrottledNanos, + responses.stream() + .map( + nr -> new ReadDetail( + nr.node.getId(), + nr.node.getName(), + nr.beforeWriteComplete, + nr.response.isNotFound(), + nr.response.getFirstByteNanos(), + nr.response.getElapsedNanos(), + nr.response.getThrottleNanos() + ) + ) + .collect(Collectors.toList()) + ) + ); + } + + /** + * @return random non-negative long in [min, max) + */ + private long randomLongBetween(long min, long max) { + assert 0 <= min && min <= max; + final long range = max - min; + return range == 0 ? min : min + (random.nextLong() & Long.MAX_VALUE) % range; + } + } + + private static class NodeResponse { + final DiscoveryNode node; + final boolean beforeWriteComplete; + final GetBlobChecksumAction.Response response; + + NodeResponse(DiscoveryNode node, boolean beforeWriteComplete, GetBlobChecksumAction.Response response) { + this.node = node; + this.beforeWriteComplete = beforeWriteComplete; + this.response = response; + } + } + + private static class WriteDetails { + final long bytesWritten; + final long elapsedNanos; + final long throttledNanos; + final long checksum; + + private WriteDetails(long bytesWritten, long elapsedNanos, long throttledNanos, long checksum) { + this.bytesWritten = bytesWritten; + this.elapsedNanos = elapsedNanos; + this.throttledNanos = throttledNanos; + this.checksum = checksum; + } + } + + public static class Request extends ActionRequest implements TaskAwareRequest { + private final String repositoryName; + private final String blobPath; + private final String blobName; + + private final long targetLength; + private final long seed; + private final List nodes; + private final boolean readEarly; + private final boolean writeAndOverwrite; + + Request( + String repositoryName, + String blobPath, + String blobName, + long targetLength, + long seed, + List nodes, + boolean readEarly, + boolean writeAndOverwrite + ) { + assert 0 < targetLength; + assert targetLength <= Integer.MAX_VALUE || (readEarly == false && writeAndOverwrite == false) : "oversized atomic write"; + this.repositoryName = repositoryName; + this.blobPath = blobPath; + this.blobName = blobName; + this.targetLength = targetLength; + this.seed = seed; + this.nodes = nodes; + this.readEarly = readEarly; + this.writeAndOverwrite = writeAndOverwrite; + } + + Request(StreamInput in) throws IOException { + super(in); + repositoryName = in.readString(); + blobPath = in.readString(); + blobName = in.readString(); + targetLength = in.readVLong(); + seed = in.readLong(); + nodes = in.readList(DiscoveryNode::new); + readEarly = in.readBoolean(); + writeAndOverwrite = in.readBoolean(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(repositoryName); + out.writeString(blobPath); + out.writeString(blobName); + out.writeVLong(targetLength); + out.writeLong(seed); + out.writeList(nodes); + out.writeBoolean(readEarly); + out.writeBoolean(writeAndOverwrite); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public String getDescription() { + return "blob speed test [" + + repositoryName + + ":" + + blobPath + + "/" + + blobName + + ", length=" + + targetLength + + ", seed=" + + seed + + ", readEarly=" + + readEarly + + ", writeAndOverwrite=" + + writeAndOverwrite + + "]"; + } + + @Override + public String toString() { + return "BlobSpeedTestAction.Request{" + getDescription() + "}"; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers) { + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + }; + } + + public String getRepositoryName() { + return repositoryName; + } + + public String getBlobPath() { + return blobPath; + } + + public String getBlobName() { + return blobName; + } + + public long getTargetLength() { + return targetLength; + } + + public long getSeed() { + return seed; + } + + } + + public static class Response extends ActionResponse implements ToXContentObject { + + private final String nodeId; + private final String nodeName; + private final String blobName; + private final long blobLength; + private final boolean readEarly; + private final boolean overwrite; + private final long checksumStart; + private final long checksumEnd; + + private final long writeElapsedNanos; + private final long overwriteElapsedNanos; + private final long writeThrottledNanos; + private final List readDetails; + + public Response( + String nodeId, + String nodeName, + String blobName, + long blobLength, + boolean readEarly, + boolean overwrite, + long checksumStart, + long checksumEnd, + long writeElapsedNanos, + long overwriteElapsedNanos, + long writeThrottledNanos, + List readDetails + ) { + this.nodeId = nodeId; + this.nodeName = nodeName; + this.blobName = blobName; + this.blobLength = blobLength; + this.readEarly = readEarly; + this.overwrite = overwrite; + this.checksumStart = checksumStart; + this.checksumEnd = checksumEnd; + this.writeElapsedNanos = writeElapsedNanos; + this.overwriteElapsedNanos = overwriteElapsedNanos; + this.writeThrottledNanos = writeThrottledNanos; + this.readDetails = readDetails; + } + + public Response(StreamInput in) throws IOException { + super(in); + nodeId = in.readString(); + nodeName = in.readString(); + blobName = in.readString(); + blobLength = in.readVLong(); + readEarly = in.readBoolean(); + overwrite = in.readBoolean(); + checksumStart = in.readVLong(); + checksumEnd = in.readVLong(); + writeElapsedNanos = in.readVLong(); + overwriteElapsedNanos = in.readVLong(); + writeThrottledNanos = in.readVLong(); + readDetails = in.readList(ReadDetail::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(nodeId); + out.writeString(nodeName); + out.writeString(blobName); + out.writeVLong(blobLength); + out.writeBoolean(readEarly); + out.writeBoolean(overwrite); + out.writeVLong(checksumStart); + out.writeVLong(checksumEnd); + out.writeVLong(writeElapsedNanos); + out.writeVLong(overwriteElapsedNanos); + out.writeVLong(writeThrottledNanos); + out.writeList(readDetails); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder.startObject("blob"); + builder.field("name", blobName); + builder.field("size", blobLength); + builder.field("read_start", checksumStart); + builder.field("read_end", checksumEnd); + builder.field("read_early", readEarly); + builder.field("overwritten", overwrite); + builder.endObject(); + + builder.startObject("writer_node"); + builder.field("id", nodeId); + builder.field("name", nodeName); + builder.endObject(); + + builder.field("write_nanos", writeElapsedNanos); + if (overwrite) { + builder.field("overwrite_nanos", overwriteElapsedNanos); + } + builder.field("write_throttled_nanos", writeThrottledNanos); + + builder.startArray("reads"); + for (ReadDetail readDetail : readDetails) { + readDetail.toXContent(builder, params); + } + builder.endArray(); + + builder.endObject(); + return builder; + } + } + + public static class ReadDetail implements Writeable, ToXContentFragment { + + private final String nodeId; + private final String nodeName; + private final boolean beforeWriteComplete; + private final boolean isNotFound; + private final long firstByteNanos; + private final long readNanos; + private final long throttleNanos; + + public ReadDetail( + String nodeId, + String nodeName, + boolean beforeWriteComplete, + boolean isNotFound, + long firstByteNanos, + long readNanos, + long throttleNanos + ) { + this.nodeId = nodeId; + this.nodeName = nodeName; + this.beforeWriteComplete = beforeWriteComplete; + this.isNotFound = isNotFound; + this.firstByteNanos = firstByteNanos; + this.readNanos = readNanos; + this.throttleNanos = throttleNanos; + } + + public ReadDetail(StreamInput in) throws IOException { + nodeId = in.readString(); + nodeName = in.readString(); + beforeWriteComplete = in.readBoolean(); + isNotFound = in.readBoolean(); + firstByteNanos = in.readVLong(); + readNanos = in.readVLong(); + throttleNanos = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(nodeId); + out.writeString(nodeName); + out.writeBoolean(beforeWriteComplete); + out.writeBoolean(isNotFound); + out.writeVLong(firstByteNanos); + out.writeVLong(readNanos); + out.writeVLong(throttleNanos); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + builder.startObject("node"); + builder.field("id", nodeId); + builder.field("name", nodeName); + builder.endObject(); + + if (beforeWriteComplete) { + builder.field("before_write_complete", true); + } + + if (isNotFound) { + builder.field("found", false); + } else { + builder.field("found", true); + builder.field("first_byte_nanos", firstByteNanos); + builder.field("read_nanos", readNanos); + builder.field("throttle_nanos", throttleNanos); + } + + builder.endObject(); + return builder; + } + } + +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java new file mode 100644 index 0000000000000..dc7576e2460bc --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java @@ -0,0 +1,353 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.RepositoryVerificationException; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskAwareRequest; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.NoSuchFileException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.CRC32; + +/** + * Action which instructs a node to read a range of a blob from a {@link org.elasticsearch.repositories.blobstore.BlobStoreRepository} + * (possibly the entire blob) and compute its checksum. It is acceptable if the blob is not found but we do not accept the blob being + * otherwise unreadable. + */ +public class GetBlobChecksumAction extends ActionType { + + private static final Logger logger = LogManager.getLogger(GetBlobChecksumAction.class); + + public static final GetBlobChecksumAction INSTANCE = new GetBlobChecksumAction(); + + public static final String NAME = "cluster:admin/repository/speed_test/blob/read"; + + private GetBlobChecksumAction() { + super(NAME, Response::new); + } + + public static class TransportAction extends HandledTransportAction { + + private static final Logger logger = GetBlobChecksumAction.logger; + + private static final int BUFFER_SIZE = ByteSizeUnit.KB.toIntBytes(8); + + private final RepositoriesService repositoriesService; + + @Inject + public TransportAction(TransportService transportService, ActionFilters actionFilters, RepositoriesService repositoriesService) { + super(NAME, transportService, actionFilters, Request::new, ThreadPool.Names.SNAPSHOT); + this.repositoriesService = repositoriesService; + } + + @Override + protected void doExecute(Task task, Request request, ActionListener listener) { + + assert task instanceof CancellableTask; + CancellableTask cancellableTask = (CancellableTask) task; + + final Repository repository = repositoriesService.repository(request.getRepositoryName()); + if (repository instanceof BlobStoreRepository == false) { + throw new IllegalArgumentException("repository [" + request.getRepositoryName() + "] is not a blob store repository"); + } + + final BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository; + final BlobContainer blobContainer = blobStoreRepository.blobStore().blobContainer(new BlobPath().add(request.getBlobPath())); + + logger.trace("handling [{}]", request); + + final InputStream rawInputStream; + try { + if (request.isWholeBlob()) { + rawInputStream = blobContainer.readBlob(request.getBlobName()); + } else { + rawInputStream = blobContainer.readBlob(request.getBlobName(), request.getRangeStart(), request.getRangeLength()); + } + } catch (FileNotFoundException | NoSuchFileException e) { + logger.trace("blob not found for [{}]", request); + listener.onResponse(Response.BLOB_NOT_FOUND); + return; + } catch (IOException e) { + logger.warn("failed to read blob for [{}]", request); + listener.onFailure(e); + return; + } + + logger.trace("reading blob for [{}]", request); + + final AtomicLong throttleNanos = new AtomicLong(); + final InputStream throttledInputStream = blobStoreRepository.maybeRateLimitRestores(rawInputStream, throttleNanos::addAndGet); + final CRC32 crc32 = new CRC32(); + final byte[] buffer = new byte[BUFFER_SIZE]; + long bytesRead = 0L; + final long startTimeNanos = System.nanoTime(); + long firstByteNanos = startTimeNanos; + + while (true) { + final int readSize; + try { + readSize = throttledInputStream.read(buffer, 0, buffer.length); + } catch (IOException e) { + logger.warn("exception while read blob for [{}]", request); + listener.onFailure(e); + return; + } + + if (readSize == -1) { + break; + } + + if (readSize > 0) { + if (bytesRead == 0L) { + firstByteNanos = System.nanoTime(); + } + + crc32.update(buffer, 0, readSize); + bytesRead += readSize; + } + + if (cancellableTask.isCancelled()) { + throw new RepositoryVerificationException( + request.repositoryName, + "cancelled [" + request.getDescription() + "] after reading [" + bytesRead + "] bytes" + ); + } + } + + final long endTimeNanos = System.nanoTime(); + + if (request.isWholeBlob() == false && bytesRead != request.getRangeLength()) { + throw new RepositoryVerificationException( + request.repositoryName, + "unexpectedly read [" + bytesRead + "] bytes when handling [" + request.getDescription() + "]" + ); + } + + final Response response = new Response( + bytesRead, + crc32.getValue(), + firstByteNanos - startTimeNanos, + endTimeNanos - startTimeNanos, + throttleNanos.get() + ); + logger.trace("responding to [{}] with [{}]", request, response); + listener.onResponse(response); + } + + } + + public static class Request extends ActionRequest implements TaskAwareRequest { + + private final String repositoryName; + private final String blobPath; + private final String blobName; + + // Requesting the range [rangeStart, rangeEnd), but we treat the range [0, 0) as a special case indicating to read the whole blob. + private final long rangeStart; + private final long rangeEnd; + + Request(StreamInput in) throws IOException { + super(in); + repositoryName = in.readString(); + blobPath = in.readString(); + blobName = in.readString(); + rangeStart = in.readVLong(); + rangeEnd = in.readVLong(); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + Request(String repositoryName, String blobPath, String blobName, long rangeStart, long rangeEnd) { + assert rangeStart == 0L && rangeEnd == 0L || 0L <= rangeStart && rangeStart < rangeEnd : rangeStart + "-" + rangeEnd; + this.repositoryName = repositoryName; + this.blobPath = blobPath; + this.blobName = blobName; + this.rangeStart = rangeStart; + this.rangeEnd = rangeEnd; + } + + public String getRepositoryName() { + return repositoryName; + } + + public String getBlobPath() { + return blobPath; + } + + public String getBlobName() { + return blobName; + } + + public long getRangeStart() { + assert isWholeBlob() == false; + return rangeStart; + } + + public long getRangeEnd() { + assert isWholeBlob() == false; + return rangeEnd; + } + + public long getRangeLength() { + assert isWholeBlob() == false; + return rangeEnd - rangeStart; + } + + /** + * @return whether we should read the whole blob or a range. + */ + boolean isWholeBlob() { + return rangeEnd == 0L; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(repositoryName); + out.writeString(blobPath); + out.writeString(blobName); + out.writeVLong(rangeStart); + out.writeVLong(rangeEnd); + } + + @Override + public String getDescription() { + return "retrieve [" + + (isWholeBlob() ? "whole blob" : (getRangeStart() + "-" + getRangeEnd())) + + "] from [" + + getRepositoryName() + + ":" + + getBlobPath() + + "/" + + getBlobName() + + "]"; + } + + @Override + public String toString() { + return "GetRepositoryBlobChecksumRequest{" + getDescription() + "}"; + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers) { + @Override + public boolean shouldCancelChildrenOnCancellation() { + return false; // no children + } + }; + } + } + + public static class Response extends ActionResponse { + + static Response BLOB_NOT_FOUND = new Response(0L, 0L, 0L, 0L, 0L); + + private final long bytesRead; // 0 if not found + private final long checksum; // 0 if not found + private final long firstByteNanos; // 0 if not found + private final long elapsedNanos; // 0 if not found + private final long throttleNanos; // 0 if not found + + Response(long bytesRead, long checksum, long firstByteNanos, long elapsedNanos, long throttleNanos) { + this.bytesRead = bytesRead; + this.checksum = checksum; + this.firstByteNanos = firstByteNanos; + this.elapsedNanos = elapsedNanos; + this.throttleNanos = throttleNanos; + } + + Response(StreamInput in) throws IOException { + super(in); + this.bytesRead = in.readVLong(); + this.checksum = in.readLong(); + this.firstByteNanos = in.readVLong(); + this.elapsedNanos = in.readVLong(); + this.throttleNanos = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(bytesRead); + out.writeLong(checksum); + out.writeVLong(firstByteNanos); + out.writeVLong(elapsedNanos); + out.writeVLong(throttleNanos); + } + + @Override + public String toString() { + return "GetRepositoryBlobChecksumResponse{" + + "bytesRead=" + + bytesRead + + ", checksum=" + + checksum + + ", firstByteNanos=" + + firstByteNanos + + ", elapsedNanos=" + + elapsedNanos + + ", throttleNanos=" + + throttleNanos + + '}'; + } + + public long getBytesRead() { + return bytesRead; + } + + public long getChecksum() { + return checksum; + } + + public long getFirstByteNanos() { + return firstByteNanos; + } + + public long getElapsedNanos() { + return elapsedNanos; + } + + public long getThrottleNanos() { + return throttleNanos; + } + + public boolean isNotFound() { + return bytesRead == 0L && checksum == 0L && firstByteNanos == 0L && elapsedNanos == 0L && throttleNanos == 0L; + } + + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java new file mode 100644 index 0000000000000..f7a1c21faa503 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.elasticsearch.repositories.RepositoryVerificationException; + +import java.util.Random; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BooleanSupplier; +import java.util.zip.CRC32; + +/** + * A random ~8kB block of data that is arbitrarily repeated, for use as a dummy payload for testing a blob store repo. + */ +class RandomBlobContent { + + static final int BUFFER_SIZE = 8191; // nearly 8kB, but prime, so unlikely to be aligned with any other block sizes + + final byte[] buffer = new byte[BUFFER_SIZE]; + private final BooleanSupplier isCancelledSupplier; + private final AtomicReference onLastRead; + private final String repositoryName; + + /** + * @param repositoryName The name of the repository being tested, for use in exception messages. + * @param seed RNG seed to use for its contents. + * @param isCancelledSupplier Predicate that causes reads to throw a {@link RepositoryVerificationException}, allowing for fast failure + * on cancellation. TODO does this actually cancel blob writes gracefully enough? + * @param onLastRead Runs when a {@code read()} call returns the last byte of the file, or on {@code close()} if the file was not fully + * read. Only runs once even if the last byte is read multiple times using {@code mark()} and {@code reset()}. + * TODO does this happen just before completion or do some SDKs read the stream more than once (e.g. checksum first)? + */ + RandomBlobContent(String repositoryName, long seed, BooleanSupplier isCancelledSupplier, Runnable onLastRead) { + this.repositoryName = repositoryName; + this.isCancelledSupplier = isCancelledSupplier; + this.onLastRead = new AtomicReference<>(onLastRead); + new Random(seed).nextBytes(buffer); + } + + long getChecksum(long checksumRangeStart, long checksumRangeEnd) { + assert 0 <= checksumRangeStart && checksumRangeStart <= checksumRangeEnd; + final CRC32 crc32 = new CRC32(); + + final long startBlock = checksumRangeStart / buffer.length; + final long endBlock = (checksumRangeEnd - 1) / buffer.length; + + if (startBlock == endBlock) { + crc32.update( + buffer, + Math.toIntExact(checksumRangeStart % buffer.length), + Math.toIntExact(checksumRangeEnd - checksumRangeStart) + ); + } else { + final int bufferStart = Math.toIntExact(checksumRangeStart % buffer.length); + crc32.update(buffer, bufferStart, buffer.length - bufferStart); + for (long block = startBlock + 1; block < endBlock; block++) { + crc32.update(buffer); + } + crc32.update(buffer, 0, Math.toIntExact((checksumRangeEnd - 1) % buffer.length) + 1); + } + + return crc32.getValue(); + } + + void ensureNotCancelled(final String position) { + if (isCancelledSupplier.getAsBoolean()) { + throw new RepositoryVerificationException(repositoryName, "blob upload cancelled at position [" + position + "]"); + } + } + + void onLastRead() { + final Runnable runnable = onLastRead.getAndSet(null); + if (runnable != null) { + runnable.run(); + } + } + +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java new file mode 100644 index 0000000000000..d6bae5524d273 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; +import org.elasticsearch.common.bytes.AbstractBytesReference; +import org.elasticsearch.common.bytes.BytesReference; + +/** + * A {@link BytesReference} that's the same random ~8kB of data repeatedly, for use as a dummy payload for testing a blob store repo. + */ +class RandomBlobContentBytesReference extends AbstractBytesReference { + + private final int length; + private final RandomBlobContent randomBlobContent; + + /** + * @param randomBlobContent The (simulated) content of the blob + * @param length The length of this blob. + */ + RandomBlobContentBytesReference(RandomBlobContent randomBlobContent, int length) { + assert 0 < length; + this.randomBlobContent = randomBlobContent; + this.length = length; + } + + @Override + public byte get(int index) { + return randomBlobContent.buffer[index % randomBlobContent.buffer.length]; + } + + @Override + public int length() { + return length; + } + + @Override + public BytesReference slice(int from, int length) { + assert false : "must not slice a RandomBlobContentBytesReference"; + throw new UnsupportedOperationException("RandomBlobContentBytesReference#slice(int, int) is unsupported"); + } + + @Override + public long ramBytesUsed() { + // no need for accurate accounting of the overhead since we don't really account for these things anyway + return randomBlobContent.buffer.length; + } + + @Override + public BytesRef toBytesRef() { + assert false : "must not materialize a RandomBlobContentBytesReference"; + throw new UnsupportedOperationException("RandomBlobContentBytesReference#toBytesRef() is unsupported"); + } + + @Override + public BytesRefIterator iterator() { + final byte[] buffer = randomBlobContent.buffer; + final int lastBlock = (length - 1) / buffer.length; + + return new BytesRefIterator() { + + int nextBlock = 0; + + @Override + public BytesRef next() { + final int block = nextBlock++; + + randomBlobContent.ensureNotCancelled(block * buffer.length + "/" + length); + final int end; + if (block == lastBlock) { + randomBlobContent.onLastRead(); + end = (length - 1) % buffer.length + 1; + } else { + end = buffer.length; + } + + return new BytesRef(buffer, 0, end); + } + }; + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java new file mode 100644 index 0000000000000..496e089c0b22a --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java @@ -0,0 +1,114 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import java.io.InputStream; + +/** + * An {@link InputStream} that's the same random ~8kB of data repeatedly, for use as a dummy payload for testing a blob store repo. + */ +class RandomBlobContentStream extends InputStream { + + private final long length; + private final RandomBlobContent randomBlobContent; + private long position; + private long markPosition; + + /** + * @param randomBlobContent The (simulated) content of the blob. + * @param length The length of this stream. + */ + RandomBlobContentStream(RandomBlobContent randomBlobContent, long length) { + assert 0 < length; + this.randomBlobContent = randomBlobContent; + this.length = length; + } + + private int bufferPosition() { + return Math.toIntExact(position % randomBlobContent.buffer.length); + } + + @Override + public int read() { + randomBlobContent.ensureNotCancelled(position + "/" + length); + + if (length <= position) { + return -1; + } + + final int b = Byte.toUnsignedInt(randomBlobContent.buffer[bufferPosition()]); + position += 1; + + if (position == length) { + randomBlobContent.onLastRead(); + } + + return b; + } + + @Override + public int read(byte[] b, int off, int len) { + randomBlobContent.ensureNotCancelled(position + "+" + len + "/" + length); + + if (length <= position) { + return -1; + } + + len = Math.toIntExact(Math.min(len, length - position)); + int remaining = len; + while (0 < remaining) { + assert position < length : position + " vs " + length; + + final int bufferPosition = bufferPosition(); + final int copied = Math.min(randomBlobContent.buffer.length - bufferPosition, remaining); + System.arraycopy(randomBlobContent.buffer, bufferPosition, b, off, copied); + off += copied; + remaining -= copied; + position += copied; + } + + assert position <= length : position + " vs " + length; + + if (position == length) { + randomBlobContent.onLastRead(); + } + + return len; + } + + @Override + public void close() { + randomBlobContent.onLastRead(); + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public synchronized void mark(int readlimit) { + markPosition = position; + } + + @Override + public synchronized void reset() { + position = markPosition; + } + + @Override + public long skip(long n) { + final long oldPosition = position; + position = Math.min(length, position + n); + return position - oldPosition; + } + + @Override + public int available() { + return Math.toIntExact(Math.min(length - position, Integer.MAX_VALUE)); + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java new file mode 100644 index 0000000000000..1b135cda00055 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java @@ -0,0 +1,718 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import com.carrotsearch.hppc.ObjectContainer; +import com.carrotsearch.hppc.cursors.ObjectCursor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.action.support.ThreadedActionListener; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobMetadata; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.xcontent.StatusToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.RepositoryVerificationException; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.tasks.CancellableTask; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; +import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.ReceiveTimeoutTransportException; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequestOptions; +import org.elasticsearch.transport.TransportResponseHandler; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.Queue; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.LongSupplier; + +/** + * Action which distributes a bunch of {@link BlobSpeedTestAction}s over the nodes in the cluster, with limited concurrency, and collects + * the results. Tries to fail fast by cancelling everything if any child task fails, or the timeout is reached, to avoid consuming + * unnecessary resources. On completion, does a best-effort wait until the blob list contains all the expected blobs, then deletes them all. + */ +public class RepositorySpeedTestAction extends ActionType { + + private static final Logger logger = LogManager.getLogger(RepositorySpeedTestAction.class); + + public static final RepositorySpeedTestAction INSTANCE = new RepositorySpeedTestAction(); + public static final String NAME = "cluster:admin/repository/speed_test"; + + private RepositorySpeedTestAction() { + super(NAME, Response::new); + } + + public static class TransportAction extends TransportMasterNodeAction { + // Needs to run on a non-voting-only master-eligible node or a data node so that it has access to the repository so it can clean up + // at the end. However today there's no good way to reroute an action onto a suitable node apart from routing it tothe elected + // master, so we run it there. This should be no big deal, coordinating the test isn't particularly heavy. + + private final RepositoriesService repositoriesService; + + @Inject + public TransportAction( + TransportService transportService, + ClusterService clusterService, + ThreadPool threadPool, + ActionFilters actionFilters, + RepositoriesService repositoriesService, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super( + NAME, + transportService, + clusterService, + threadPool, + actionFilters, + RepositorySpeedTestAction.Request::new, + indexNameExpressionResolver, + RepositorySpeedTestAction.Response::new, + ThreadPool.Names.SAME + ); + this.repositoriesService = repositoriesService; + } + + @Override + protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) { + final Repository repository = repositoriesService.repository(request.getRepositoryName()); + if (repository instanceof BlobStoreRepository == false) { + throw new IllegalArgumentException("repository [" + request.getRepositoryName() + "] is not a blob-store repository"); + } + if (repository.isReadOnly()) { + throw new IllegalArgumentException("repository [" + request.getRepositoryName() + "] is read-only"); + } + + request.reseed(threadPool.relativeTimeInMillis()); + + assert task instanceof CancellableTask; + new AsyncAction( + transportService, + (BlobStoreRepository) repository, + (CancellableTask) task, + request, + state.nodes(), + threadPool::relativeTimeInMillis, + listener + ).run(); + } + + @Override + protected ClusterBlockException checkBlock(Request request, ClusterState state) { + return null; + } + } + + public static class AsyncAction { + + private final TransportService transportService; + private final BlobStoreRepository repository; + private final CancellableTask task; + private final Request request; + private final DiscoveryNodes discoveryNodes; + private final LongSupplier currentTimeMillisSupplier; + private final ActionListener listener; + private final long timeoutTimeMillis; + + // choose the blob path nondeterministically to avoid clashes, assuming that the actual path doesn't matter for reproduction + private final String blobPath = "temp-speed-test-" + UUIDs.randomBase64UUID(); + + private final Queue queue = ConcurrentCollections.newQueue(); + private final AtomicReference failure = new AtomicReference<>(); + private final Semaphore innerFailures = new Semaphore(5); // limit the number of suppressed failures + private final int workerCount; + private final GroupedActionListener workersListener; + private final Set expectedBlobs = ConcurrentCollections.newConcurrentSet(); + private final List responses; + + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + private OptionalLong listingStartTimeNanos = OptionalLong.empty(); + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + private OptionalLong deleteStartTimeNanos = OptionalLong.empty(); + + public AsyncAction( + TransportService transportService, + BlobStoreRepository repository, + CancellableTask task, + Request request, + DiscoveryNodes discoveryNodes, + LongSupplier currentTimeMillisSupplier, + ActionListener listener + ) { + this.transportService = transportService; + this.repository = repository; + this.task = task; + this.request = request; + this.discoveryNodes = discoveryNodes; + this.currentTimeMillisSupplier = currentTimeMillisSupplier; + this.timeoutTimeMillis = currentTimeMillisSupplier.getAsLong() + request.getTimeout().millis(); + this.listener = listener; + + this.workerCount = request.getConcurrency(); + this.workersListener = new GroupedActionListener<>( + new ThreadedActionListener<>(logger, transportService.getThreadPool(), ThreadPool.Names.SNAPSHOT, new ActionListener<>() { + @Override + public void onResponse(Collection voids) { + onWorkerCompletion(); + } + + @Override + public void onFailure(Exception e) { + assert false : e; // workers should not fail + onWorkerCompletion(); + } + }, false), + workerCount + ); + + responses = new ArrayList<>(request.blobCount); + } + + private void fail(Exception e) { + if (failure.compareAndSet(null, e)) { + transportService.getTaskManager().cancelTaskAndDescendants(task, "task failed", false, ActionListener.wrap(() -> {})); + } else { + if (innerFailures.tryAcquire()) { + final Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof TaskCancelledException || cause instanceof ReceiveTimeoutTransportException) { + innerFailures.release(); + } else { + failure.get().addSuppressed(e); + } + } + } + } + + /** + * Check that we haven't already failed or been cancelled or timed out; if newly cancelled or timed out then record this as the root + * cause of failure. + */ + private boolean isRunning() { + if (failure.get() != null) { + return false; + } + + if (task.isCancelled()) { + failure.compareAndSet(null, new RepositoryVerificationException(request.repositoryName, "verification cancelled")); + // if this CAS failed then we're failing for some other reason, nbd; also if the task is cancelled then its descendants are + // also cancelled, so no further action is needed either way. + return false; + } + + if (timeoutTimeMillis < currentTimeMillisSupplier.getAsLong()) { + if (failure.compareAndSet( + null, + new RepositoryVerificationException(request.repositoryName, "speed test timed out after [" + request.getTimeout() + "]") + )) { + transportService.getTaskManager().cancelTaskAndDescendants(task, "timed out", false, ActionListener.wrap(() -> {})); + } + // if this CAS failed then we're already failing for some other reason, nbd + return false; + } + + return true; + } + + public void run() { + assert queue.isEmpty() && failure.get() == null : "must only run action once"; + + logger.info("running speed test of repository [{}] using path [{}]", request.getRepositoryName(), blobPath); + + final Random random = new Random(request.getSeed()); + + final ObjectContainer nodesContainer = discoveryNodes.getMasterAndDataNodes().values(); + final List nodes = new ArrayList<>(nodesContainer.size()); + for (ObjectCursor cursor : nodesContainer) { + nodes.add(cursor.value); + } + + final long maxBlobSize = request.getMaxBlobSize().getBytes(); + final List blobSizes = new ArrayList<>(); + for (long s = 1; s < maxBlobSize; s <<= 1) { + blobSizes.add(s); + } + blobSizes.add(maxBlobSize); + + for (int i = 0; i < request.getBlobCount(); i++) { + final long targetLength = blobSizes.get(random.nextInt(blobSizes.size())); + final boolean smallBlob = targetLength <= Integer.MAX_VALUE; // we only use the non-atomic API for larger blobs + final VerifyBlobTask verifyBlobTask = new VerifyBlobTask( + nodes.get(random.nextInt(nodes.size())), + new BlobSpeedTestAction.Request( + request.getRepositoryName(), + blobPath, + "speed-test-" + i + "-" + UUIDs.randomBase64UUID(random), + targetLength, + random.nextLong(), + nodes, + smallBlob && (random.nextInt(50) == 0), // TODO magic 50 + repository.supportURLRepo() && smallBlob && random.nextInt(50) == 0 + ) + ); // TODO magic 50 + queue.add(verifyBlobTask); + } + + for (int i = 0; i < workerCount; i++) { + processNextTask(); + } + } + + private void processNextTask() { + final VerifyBlobTask thisTask = queue.poll(); + if (isRunning() == false || thisTask == null) { + workersListener.onResponse(null); + } else { + logger.trace("processing [{}]", thisTask); + // NB although all this is on the SAME thread, the per-blob verification runs on a SNAPSHOT thread so we don't have to worry + // about local requests resulting in a stack overflow here + final TransportRequestOptions transportRequestOptions = TransportRequestOptions.timeout( + TimeValue.timeValueMillis(timeoutTimeMillis - currentTimeMillisSupplier.getAsLong()) + ); + transportService.sendChildRequest( + thisTask.node, + BlobSpeedTestAction.NAME, + thisTask.request, + task, + transportRequestOptions, + new TransportResponseHandler() { + @Override + public void handleResponse(BlobSpeedTestAction.Response response) { + logger.trace("finished [{}]", thisTask); + expectedBlobs.add(thisTask.request.getBlobName()); // each task cleans up its own mess on failure + synchronized (responses) { + responses.add(response); + } + processNextTask(); + } + + @Override + public void handleException(TransportException exp) { + logger.debug(new ParameterizedMessage("failed [{}]", thisTask), exp); + fail(exp); + workersListener.onResponse(null); + } + + @Override + public BlobSpeedTestAction.Response read(StreamInput in) throws IOException { + return new BlobSpeedTestAction.Response(in); + } + } + ); + } + + } + + private BlobContainer getBlobContainer() { + return repository.blobStore().blobContainer(new BlobPath().add(blobPath)); + } + + private void onWorkerCompletion() { + tryListAndCleanup(0); + } + + private void tryListAndCleanup(final int retryCount) { + if (timeoutTimeMillis < currentTimeMillisSupplier.getAsLong() || task.isCancelled()) { + logger.warn( + "speed test of repository [{}] failed in cleanup phase after [{}] attempts, attempting best-effort cleanup " + + "but you may need to manually remove [{}]", + request.getRepositoryName(), + retryCount, + blobPath + ); + isRunning(); // set failure if not already set + deleteContainerAndSendResponse(); + } else { + boolean retry = true; + try { + if (listingStartTimeNanos.isEmpty()) { + assert retryCount == 0 : retryCount; + logger.trace( + "all tasks completed, checking expected blobs exist in [{}:{}] before cleanup", + request.repositoryName, + blobPath + ); + listingStartTimeNanos = OptionalLong.of(System.nanoTime()); + } else { + logger.trace( + "retrying check that expected blobs exist in [{}:{}] before cleanup, retry count = {}", + request.repositoryName, + blobPath, + retryCount + ); + } + final BlobContainer blobContainer = getBlobContainer(); + final Map blobsMap = blobContainer.listBlobs(); + + final Set missingBlobs = new HashSet<>(expectedBlobs); + missingBlobs.removeAll(blobsMap.keySet()); + if (missingBlobs.isEmpty()) { + logger.trace( + "all expected blobs found after {} failed attempts, cleaning up [{}:{}]", + retryCount, + request.getRepositoryName(), + blobPath + ); + retry = false; + deleteContainerAndSendResponse(); + } else { + logger.debug( + "expected blobs [{}] missing in [{}:{}], trying again; retry count = {}", + request.repositoryName, + blobPath, + missingBlobs, + retryCount + ); + } + } catch (Exception e) { + assert retry; + logger.debug( + new ParameterizedMessage( + "failure during cleanup of [{}:{}], will retry; retry count = {}", + request.getRepositoryName(), + blobPath, + retryCount + ), + e + ); + fail(e); + } finally { + if (retry) { + // Either there were missing blobs, or else listing the blobs failed. + transportService.getThreadPool() + .scheduleUnlessShuttingDown( + TimeValue.timeValueSeconds(10), + ThreadPool.Names.SNAPSHOT, + () -> tryListAndCleanup(retryCount + 1) + ); + } + } + } + } + + private void deleteContainerAndSendResponse() { + try { + assert deleteStartTimeNanos.isEmpty() : deleteStartTimeNanos; + deleteStartTimeNanos = OptionalLong.of(System.nanoTime()); + getBlobContainer().delete(); + } catch (Exception e) { + fail(e); + } finally { + sendResponse(); + } + } + + private void sendResponse() { + final Exception exception = failure.get(); + if (exception == null) { + assert listingStartTimeNanos.isPresent(); + assert deleteStartTimeNanos.isPresent(); + final long completionTimeNanos = System.nanoTime(); + + logger.trace("[{}] completed successfully", request.getDescription()); + + listener.onResponse( + new Response( + request.getRepositoryName(), + request.blobCount, + request.concurrency, + request.maxBlobSize, + request.seed, + blobPath, + responses, + deleteStartTimeNanos.getAsLong() - listingStartTimeNanos.getAsLong(), + completionTimeNanos - deleteStartTimeNanos.getAsLong() + ) + ); + } else { + logger.debug(new ParameterizedMessage("speed test of repository [{}] failed", request.repositoryName), exception); + listener.onFailure( + new RepositoryVerificationException( + request.getRepositoryName(), + "speed test failed, you may need to manually remove [" + blobPath + "]", + exception + ) + ); + } + } + + private static class VerifyBlobTask { + final DiscoveryNode node; + final BlobSpeedTestAction.Request request; + + VerifyBlobTask(DiscoveryNode node, BlobSpeedTestAction.Request request) { + this.node = node; + this.request = request; + } + + @Override + public String toString() { + return "VerifyBlobTask{" + "node=" + node + ", request=" + request + '}'; + } + } + } + + public static class Request extends MasterNodeRequest { + + private final String repositoryName; + + private int blobCount = 100; + private int concurrency = 10; + private long seed = 0L; + private TimeValue timeout = TimeValue.timeValueSeconds(30); + + private ByteSizeValue maxBlobSize = ByteSizeValue.ofMb(10); + + public Request(String repositoryName) { + this.repositoryName = repositoryName; + } + + public Request(StreamInput in) throws IOException { + super(in); + repositoryName = in.readString(); + seed = in.readLong(); + blobCount = in.readVInt(); + concurrency = in.readVInt(); + timeout = in.readTimeValue(); + maxBlobSize = new ByteSizeValue(in); + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(repositoryName); + out.writeLong(seed); + out.writeVInt(blobCount); + out.writeVInt(concurrency); + out.writeTimeValue(timeout); + maxBlobSize.writeTo(out); + } + + @Override + public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers) { + @Override + public boolean shouldCancelChildrenOnCancellation() { + return true; + } + }; + } + + public void blobCount(int blobCount) { + if (blobCount <= 0) { + throw new IllegalArgumentException("blobCount must be >0, but was [" + blobCount + "]"); + } + this.blobCount = blobCount; + } + + public void concurrency(int concurrency) { + if (concurrency <= 0) { + throw new IllegalArgumentException("concurrency must be >0, but was [" + concurrency + "]"); + } + this.concurrency = concurrency; + } + + public void seed(long seed) { + this.seed = seed; + } + + public void timeout(TimeValue timeout) { + this.timeout = timeout; + } + + public void maxBlobSize(ByteSizeValue maxBlobSize) { + if (maxBlobSize.getBytes() <= 0) { + throw new IllegalArgumentException("maxBlobSize must be >0, but was [" + maxBlobSize + "]"); + } + this.maxBlobSize = maxBlobSize; + } + + public int getBlobCount() { + return blobCount; + } + + public int getConcurrency() { + return concurrency; + } + + public String getRepositoryName() { + return repositoryName; + } + + public TimeValue getTimeout() { + return timeout; + } + + public long getSeed() { + return seed; + } + + public ByteSizeValue getMaxBlobSize() { + return maxBlobSize; + } + + @Override + public String toString() { + return "Request{" + getDescription() + '}'; + } + + @Override + public String getDescription() { + return "speed test [repository=" + + repositoryName + + ", blobCount=" + + blobCount + + ", concurrency=" + + concurrency + + ", seed=" + + seed + + ", timeout=" + + timeout + + ", maxBlobSize=" + + maxBlobSize + + "]"; + } + + public void reseed(long newSeed) { + if (seed == 0L) { + seed = newSeed; + } + } + } + + public static class Response extends ActionResponse implements StatusToXContentObject { + + private final String repositoryName; + private final int blobCount; + private final int concurrency; + private final ByteSizeValue maxBlobSize; + private final long seed; + private final String blobPath; + private final List blobResponses; + private final long listingTimeNanos; + private final long deleteTimeNanos; + + public Response( + String repositoryName, + int blobCount, + int concurrency, + ByteSizeValue maxBlobSize, + long seed, + String blobPath, + List blobResponses, + long listingTimeNanos, + long deleteTimeNanos + ) { + this.repositoryName = repositoryName; + this.blobCount = blobCount; + this.concurrency = concurrency; + this.maxBlobSize = maxBlobSize; + this.seed = seed; + this.blobPath = blobPath; + this.blobResponses = blobResponses; + this.listingTimeNanos = listingTimeNanos; + this.deleteTimeNanos = deleteTimeNanos; + } + + public Response(StreamInput in) throws IOException { + super(in); + repositoryName = in.readString(); + blobCount = in.readVInt(); + concurrency = in.readVInt(); + maxBlobSize = new ByteSizeValue(in); + seed = in.readLong(); + blobPath = in.readString(); + blobResponses = in.readList(BlobSpeedTestAction.Response::new); + listingTimeNanos = in.readVLong(); + deleteTimeNanos = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(repositoryName); + out.writeVInt(blobCount); + out.writeVInt(concurrency); + maxBlobSize.writeTo(out); + out.writeLong(seed); + out.writeString(blobPath); + out.writeList(blobResponses); + out.writeVLong(listingTimeNanos); + out.writeVLong(deleteTimeNanos); + } + + @Override + public RestStatus status() { + return RestStatus.OK; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("repository", repositoryName); + builder.field("blob_count", blobCount); + builder.field("concurrency", concurrency); + builder.field("max_blob_size", maxBlobSize); + builder.field("seed", seed); + builder.field("blob_path", blobPath); + + builder.startArray("blobs"); + for (BlobSpeedTestAction.Response blobResponse : blobResponses) { + blobResponse.toXContent(builder, params); + } + builder.endArray(); + + builder.field("listing_nanos", listingTimeNanos); + builder.field("delete_nanos", deleteTimeNanos); + + builder.endObject(); + return builder; + } + } + +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java new file mode 100644 index 0000000000000..b899a44486fbb --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.rest.action.admin.cluster.RestRepositorySpeedTestAction; + +import java.util.List; +import java.util.function.Supplier; + +public class SnapshotRepositoryTestKit extends Plugin implements ActionPlugin { + + @Override + public List> getActions() { + return List.of( + new ActionHandler<>(RepositorySpeedTestAction.INSTANCE, RepositorySpeedTestAction.TransportAction.class), + new ActionHandler<>(BlobSpeedTestAction.INSTANCE, BlobSpeedTestAction.TransportAction.class), + new ActionHandler<>(GetBlobChecksumAction.INSTANCE, GetBlobChecksumAction.TransportAction.class) + ); + } + + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + return List.of(new RestRepositorySpeedTestAction()); + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRepositorySpeedTestAction.java new file mode 100644 index 0000000000000..a522cdabb868c --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRepositorySpeedTestAction.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.rest.action.admin.cluster; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.repositories.blobstore.testkit.RepositorySpeedTestAction; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestStatusToXContentListener; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class RestRepositorySpeedTestAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new Route(POST, "/_snapshot/{repository}/_speed_test")); + } + + @Override + public String getName() { + return "repository_speed_test"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + RepositorySpeedTestAction.Request verifyRepositoryRequest = new RepositorySpeedTestAction.Request(request.param("repository")); + + verifyRepositoryRequest.blobCount(request.paramAsInt("blob_count", verifyRepositoryRequest.getBlobCount())); + verifyRepositoryRequest.concurrency(request.paramAsInt("concurrency", verifyRepositoryRequest.getConcurrency())); + verifyRepositoryRequest.seed(request.paramAsLong("seed", verifyRepositoryRequest.getSeed())); + verifyRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", verifyRepositoryRequest.getMaxBlobSize())); + verifyRepositoryRequest.timeout(request.paramAsTime("timeout", verifyRepositoryRequest.getTimeout())); + verifyRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", verifyRepositoryRequest.masterNodeTimeout())); + + RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); + return channel -> cancelClient.execute( + RepositorySpeedTestAction.INSTANCE, + verifyRepositoryRequest, + new RestStatusToXContentListener<>(channel) + ); + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java new file mode 100644 index 0000000000000..fa1397262a094 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.apache.http.client.methods.HttpPost; +import org.elasticsearch.client.Request; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.test.rest.ESRestTestCase; + +public abstract class AbstractSnapshotRepoTestKitRestTestCase extends ESRestTestCase { + + protected abstract String repositoryType(); + + protected abstract Settings repositorySettings(); + + public void testRepositorySpeedTest() throws Exception { + final String repositoryType = repositoryType(); + final Settings repositorySettings = repositorySettings(); + + final String repository = "repository"; + logger.info("creating repository [{}] of type [{}]", repository, repositoryType); + registerRepository(repository, repositoryType, true, repositorySettings); + + final Request request = new Request(HttpPost.METHOD_NAME, "/_snapshot/" + repository + "/_speed_test"); + request.addParameter("blob_count", "10"); + request.addParameter("concurrency", "4"); + request.addParameter("max_blob_size", "1mb"); + request.addParameter("timeout", "120s"); + assertOK(client().performRequest(request)); + } + +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java new file mode 100644 index 0000000000000..5d37b2fe2278e --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.elasticsearch.repositories.RepositoryVerificationException; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.CRC32; + +import static org.elasticsearch.repositories.blobstore.testkit.RandomBlobContent.BUFFER_SIZE; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +public class RandomBlobContentStreamTests extends ESTestCase { + + public void testReadAndResetAndSkip() { + + final byte[] output = new byte[randomSize()]; + final AtomicBoolean readComplete = new AtomicBoolean(); + int position = 0; + int mark = 0; + int resetCount = 0; + + final int checksumStart = between(0, output.length - 1); + final int checksumEnd = between(checksumStart, output.length); + final long checksum; + + final RandomBlobContent randomBlobContent = new RandomBlobContent( + "repo", + randomLong(), + () -> false, + () -> assertTrue("multiple notifications", readComplete.compareAndSet(false, true)) + ); + + try (RandomBlobContentStream randomBlobContentStream = new RandomBlobContentStream(randomBlobContent, output.length)) { + + assertTrue(randomBlobContentStream.markSupported()); + + checksum = randomBlobContent.getChecksum(checksumStart, checksumEnd); + + while (readComplete.get() == false) { + switch (between(1, resetCount < 10 ? 4 : 3)) { + case 1: + randomBlobContentStream.mark(between(0, Integer.MAX_VALUE)); + mark = position; + break; + case 2: + final int nextByte = randomBlobContentStream.read(); + assertThat(nextByte, not(equalTo(-1))); + output[position++] = (byte) nextByte; + break; + case 3: + final int len = between(0, output.length - position); + assertThat(randomBlobContentStream.read(output, position, len), equalTo(len)); + position += len; + break; + case 4: + randomBlobContentStream.reset(); + resetCount += 1; + if (randomBoolean()) { + final int skipBytes = between(0, position - mark); + assertThat(randomBlobContentStream.skip(skipBytes), equalTo((long) skipBytes)); + position = mark + skipBytes; + } else { + position = mark; + } + break; + } + } + + if (randomBoolean()) { + assertThat(randomBlobContentStream.read(), equalTo(-1)); + } else { + assertThat(randomBlobContentStream.read(output, 0, between(0, output.length)), equalTo(-1)); + } + } + + for (int i = BUFFER_SIZE; i < output.length; i++) { + assertThat("output repeats at position " + i, output[i], equalTo(output[i % BUFFER_SIZE])); + } + + final CRC32 crc32 = new CRC32(); + crc32.update(output, checksumStart, checksumEnd - checksumStart); + assertThat("checksum computed correctly", checksum, equalTo(crc32.getValue())); + } + + public void testNotifiesOfCompletionOnce() throws IOException { + + final AtomicBoolean readComplete = new AtomicBoolean(); + final RandomBlobContent randomBlobContent = new RandomBlobContent( + "repo", + randomLong(), + () -> false, + () -> assertTrue("multiple notifications", readComplete.compareAndSet(false, true)) + ); + + try (RandomBlobContentStream randomBlobContentStream = new RandomBlobContentStream(randomBlobContent, randomSize())) { + for (int i = between(0, 4); i > 0; i--) { + randomBlobContentStream.reset(); + randomBlobContentStream.readAllBytes(); + assertTrue(readComplete.get()); + } + } + + assertTrue(readComplete.get()); // even if not completely read + } + + public void testReadingOneByteThrowsExceptionAfterCancellation() { + final RandomBlobContent randomBlobContent = new RandomBlobContent("repo", randomLong(), () -> true, () -> {}); + + try (RandomBlobContentStream randomBlobContentStream = new RandomBlobContentStream(randomBlobContent, randomSize())) { + // noinspection ResultOfMethodCallIgnored + expectThrows(RepositoryVerificationException.class, randomBlobContentStream::read); + } + } + + public void testReadingBytesThrowsExceptionAfterCancellation() { + final RandomBlobContent randomBlobContent = new RandomBlobContent("repo", randomLong(), () -> true, () -> {}); + + try (RandomBlobContentStream randomBlobContentStream = new RandomBlobContentStream(randomBlobContent, randomSize())) { + expectThrows(RepositoryVerificationException.class, randomBlobContentStream::readAllBytes); + } + } + + private static int randomSize() { + return between(0, 30000); + } + +} From 6dc193c97588397bb4e8460bcac8f08ac7ac4b78 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 12 Jan 2021 08:46:43 +0000 Subject: [PATCH 02/43] Include blob-level request in more failure messages --- .../testkit/BlobSpeedTestAction.java | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java index 8b6a5d1743ee4..ca69d027e1e49 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java @@ -269,16 +269,39 @@ private void readOnNodes(List nodes, boolean beforeWriteComplete) } else { // no need for extra synchronization after checking if we were cancelled a couple of lines ago -- we haven't notified // the outer listener yet so any bans on the children are still in place + final GetBlobChecksumAction.Request blobChecksumRequest = getBlobChecksumRequest(); transportService.sendChildRequest( node, GetBlobChecksumAction.NAME, - getBlobChecksumRequest(), + blobChecksumRequest, task, TransportRequestOptions.EMPTY, - new ActionListenerResponseHandler<>( - readNodesListener.map(r -> makeNodeResponse(node, beforeWriteComplete, r)), - GetBlobChecksumAction.Response::new - ) + new ActionListenerResponseHandler<>(new ActionListener<>() { + @Override + public void onResponse(GetBlobChecksumAction.Response response) { + readNodesListener.onResponse(makeNodeResponse(node, beforeWriteComplete, response)); + } + + @Override + public void onFailure(Exception e) { + readNodesListener.onFailure( + new RepositoryVerificationException( + request.getRepositoryName(), + "[" + + blobChecksumRequest + + "] (" + + (beforeWriteComplete ? "before" : "after") + + " write complete) failed on node [" + + node + + "] as part of [" + + request.getDescription() + + "]", + e + ) + ); + + } + }, GetBlobChecksumAction.Response::new) ); } } @@ -389,6 +412,8 @@ private void onReadsComplete(Collection responses, WriteDetails wr + expectedChecksumDescription + "] but read [" + response + + "] as part of [" + + request.getDescription() + "]" ); } From 5f4837613530c79fcf530db96f08b969ecafdb80 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 12 Jan 2021 08:48:12 +0000 Subject: [PATCH 03/43] Everyone loves a sequence diagram Courtesy of http://weidagang.github.io/text-diagram/ with this input: object Writer Repo Readers note right of Writer: Write phase Writer -> Repo: Write blob with random content Writer -> Readers: Read range during write (rarely) Readers -> Repo: Read range Repo -> Readers: Contents of range, or "not found" Readers -> Writer: Acknowledge read, including checksum if found Repo -> Writer: Write complete space 5 note right of Writer: Read phase Writer -> Readers: Read range [a,b) Readers -> Repo: Read range Writer -> Repo: Overwrite blob (rarely) Repo -> Readers: Contents of range Repo -> Writer: Overwrite complete Readers -> Writer: Ack read (with checksum) space 5 note right of Writer: Verify phase Writer -> Writer: Confirm checksums --- .../testkit/BlobSpeedTestAction.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java index ca69d027e1e49..2c1b062b4e0d9 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java @@ -64,8 +64,79 @@ * version of the blob, but again must not yield partial data). Usually, however, we write once and only read after the write completes, and * in this case we insist that the read succeeds. * + * + * + * + * +---------+ +-------+ +---------+ + * | Writer | | Repo | | Readers | + * +---------+ +-------+ +---------+ + * | --------------\ | | + * |-| Write phase | | | + * | |-------------| | | + * | | | + * | Write blob with random content | | + * |----------------------------------->| | + * | | | + * | Read range during write (rarely) | | + * |---------------------------------------------------------------------------->| + * | | | + * | | Read range | + * | |<---------------------------------------| + * | | | + * | | Contents of range, or "not found" | + * | |--------------------------------------->| + * | | | + * | Acknowledge read, including checksum if found | + * |<----------------------------------------------------------------------------| + * | | | + * | Write complete | | + * |<-----------------------------------| | + * | | | + * | | | + * | | | + * | | | + * | | | + * | -------------\ | | + * |-| Read phase | | | + * | |------------| | | + * | | | + * | Read range [a,b) | | + * |---------------------------------------------------------------------------->| + * | | | + * | | Read range | + * | |<---------------------------------------| + * | | | + * | Overwrite blob (rarely) | | + * |----------------------------------->| | + * | | | + * | | Contents of range | + * | |--------------------------------------->| + * | | | + * | Overwrite complete | | + * |<-----------------------------------| | + * | | | + * | | Ack read (with checksum) | + * |<----------------------------------------------------------------------------| + * | | | + * | | | + * | | | + * | | | + * | | | + * | ---------------\ | | + * |-| Verify phase | | | + * | |--------------| | | + * | | | + * | Confirm checksums | | + * |------------------ | | + * | | | | + * |<----------------- | | + * | | | + * + * + * * On success, details of how long everything took are returned. On failure, cancels the remote read tasks to try and avoid consuming * unnecessary resources. + * */ public class BlobSpeedTestAction extends ActionType { From d96d5ad5c69bd63a5c2caa8e7bfaca066002f71e Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 11:22:37 +0000 Subject: [PATCH 04/43] Apparently precommit doesn't like a sequence diagram --- .../testkit/BlobSpeedTestAction.java | 26 +++++++++---------- .../RestRepositorySpeedTestAction.java | 3 ++- 2 files changed, 15 insertions(+), 14 deletions(-) rename x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/{rest/action/admin/cluster => repositories/blobstore/testkit}/RestRepositorySpeedTestAction.java (93%) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java index 2c1b062b4e0d9..f1ac99d45bc75 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java @@ -75,22 +75,22 @@ * | |-------------| | | * | | | * | Write blob with random content | | - * |----------------------------------->| | + * |-----------------------------------→| | * | | | * | Read range during write (rarely) | | - * |---------------------------------------------------------------------------->| + * |----------------------------------------------------------------------------→| * | | | * | | Read range | - * | |<---------------------------------------| + * | |←---------------------------------------| * | | | * | | Contents of range, or "not found" | - * | |--------------------------------------->| + * | |---------------------------------------→| * | | | * | Acknowledge read, including checksum if found | - * |<----------------------------------------------------------------------------| + * |←----------------------------------------------------------------------------| * | | | * | Write complete | | - * |<-----------------------------------| | + * |←-----------------------------------| | * | | | * | | | * | | | @@ -101,22 +101,22 @@ * | |------------| | | * | | | * | Read range [a,b) | | - * |---------------------------------------------------------------------------->| + * |----------------------------------------------------------------------------→| * | | | * | | Read range | - * | |<---------------------------------------| + * | |←---------------------------------------| * | | | * | Overwrite blob (rarely) | | - * |----------------------------------->| | + * |-----------------------------------→| | * | | | * | | Contents of range | - * | |--------------------------------------->| + * | |---------------------------------------→| * | | | * | Overwrite complete | | - * |<-----------------------------------| | + * |←-----------------------------------| | * | | | * | | Ack read (with checksum) | - * |<----------------------------------------------------------------------------| + * |←----------------------------------------------------------------------------| * | | | * | | | * | | | @@ -129,7 +129,7 @@ * | Confirm checksums | | * |------------------ | | * | | | | - * |<----------------- | | + * |←----------------- | | * | | | * * diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java similarity index 93% rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRepositorySpeedTestAction.java rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java index a522cdabb868c..1d4e0aed4d0a3 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestRepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java @@ -31,7 +31,7 @@ public String getName() { } @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { RepositorySpeedTestAction.Request verifyRepositoryRequest = new RepositorySpeedTestAction.Request(request.param("repository")); verifyRepositoryRequest.blobCount(request.paramAsInt("blob_count", verifyRepositoryRequest.getBlobCount())); @@ -40,6 +40,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC verifyRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", verifyRepositoryRequest.getMaxBlobSize())); verifyRepositoryRequest.timeout(request.paramAsTime("timeout", verifyRepositoryRequest.getTimeout())); verifyRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", verifyRepositoryRequest.masterNodeTimeout())); + verifyRepositoryRequest.detailed(request.paramAsBoolean("detailed", verifyRepositoryRequest.getDetailed())); RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); return channel -> cancelClient.execute( From b744d9ec2c6676fe4b4f02787a66f22d7bfde002 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 11:25:20 +0000 Subject: [PATCH 05/43] Add detailed parameter --- .../api/snapshot.repository_speed_test.json | 4 ++ .../rest-api-spec/test/10_speed_test.yml | 29 ++++++++++++- .../testkit/RepositorySpeedTestAction.java | 42 ++++++++++++------- .../RestRepositorySpeedTestAction.java | 4 +- .../testkit/SnapshotRepositoryTestKit.java | 1 - .../testkit/SpeedTestStatistics.java | 11 +++++ 6 files changed, 70 insertions(+), 21 deletions(-) create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json index 37fb28760a971..01db6c20e4d38 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json @@ -46,6 +46,10 @@ "master_timeout":{ "type":"time", "description":"Explicit operation timeout for connection to master node" + }, + "detailed":{ + "type":"boolean", + "description":"Whether to return detailed results or a summary. Defaults to `false`." } } } diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml index 0c688dd2abf85..9e4f21d885005 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml @@ -33,8 +33,32 @@ setup: - match: { error.type: illegal_argument_exception } - match: { error.reason: "repository [test_repo_readonly] is read-only" } + +--- +"Speed test without details": + - skip: + version: "- 7.99.99" + reason: "introduced in 8.0" + + - do: + snapshot.repository_speed_test: + repository: test_repo + blob_count: 10 + concurrency: 5 + max_blob_size: 1mb + + - match: { repository: test_repo } + - match: { blob_count: 10 } + - match: { concurrency: 5 } + - match: { max_blob_size: 1mb } + - is_true: seed + - is_true: blob_path + - is_false: details + - gte: { listing_nanos: 0} + - gte: { delete_nanos: 0} + --- -"Speed test": +"Speed test with details": - skip: version: "- 7.99.99" reason: "introduced in 8.0" @@ -45,6 +69,7 @@ setup: blob_count: 10 concurrency: 5 max_blob_size: 1mb + detailed: true - match: { repository: test_repo } - match: { blob_count: 10 } @@ -52,6 +77,6 @@ setup: - match: { max_blob_size: 1mb } - is_true: seed - is_true: blob_path - - is_true: blobs + - is_true: details - gte: { listing_nanos: 0} - gte: { delete_nanos: 0} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java index 1b135cda00055..9746e6d227887 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java @@ -166,6 +166,7 @@ public static class AsyncAction { private final GroupedActionListener workersListener; private final Set expectedBlobs = ConcurrentCollections.newConcurrentSet(); private final List responses; + private final SpeedTestStatistics statistics = new SpeedTestStatistics(); @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private OptionalLong listingStartTimeNanos = OptionalLong.empty(); @@ -288,9 +289,9 @@ public void run() { random.nextLong(), nodes, smallBlob && (random.nextInt(50) == 0), // TODO magic 50 - repository.supportURLRepo() && smallBlob && random.nextInt(50) == 0 + repository.supportURLRepo() && smallBlob && random.nextInt(50) == 0 // TODO magic 50 ) - ); // TODO magic 50 + ); queue.add(verifyBlobTask); } @@ -321,8 +322,10 @@ private void processNextTask() { public void handleResponse(BlobSpeedTestAction.Response response) { logger.trace("finished [{}]", thisTask); expectedBlobs.add(thisTask.request.getBlobName()); // each task cleans up its own mess on failure - synchronized (responses) { - responses.add(response); + if (request.detailed) { + synchronized (responses) { + responses.add(response); + } } processNextTask(); } @@ -501,8 +504,8 @@ public static class Request extends MasterNodeRequest { private int concurrency = 10; private long seed = 0L; private TimeValue timeout = TimeValue.timeValueSeconds(30); - private ByteSizeValue maxBlobSize = ByteSizeValue.ofMb(10); + private boolean detailed = false; public Request(String repositoryName) { this.repositoryName = repositoryName; @@ -516,6 +519,7 @@ public Request(StreamInput in) throws IOException { concurrency = in.readVInt(); timeout = in.readTimeValue(); maxBlobSize = new ByteSizeValue(in); + detailed = in.readBoolean(); } @Override @@ -532,16 +536,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(concurrency); out.writeTimeValue(timeout); maxBlobSize.writeTo(out); + out.writeBoolean(detailed); } @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers) { - @Override - public boolean shouldCancelChildrenOnCancellation() { - return true; - } - }; + return new CancellableTask(id, type, action, getDescription(), parentTaskId, headers); } public void blobCount(int blobCount) { @@ -573,6 +573,10 @@ public void maxBlobSize(ByteSizeValue maxBlobSize) { this.maxBlobSize = maxBlobSize; } + public void detailed(boolean detailed) { + this.detailed = detailed; + } + public int getBlobCount() { return blobCount; } @@ -597,6 +601,10 @@ public ByteSizeValue getMaxBlobSize() { return maxBlobSize; } + public boolean getDetailed() { + return detailed; + } + @Override public String toString() { return "Request{" + getDescription() + '}'; @@ -616,6 +624,8 @@ public String getDescription() { + timeout + ", maxBlobSize=" + maxBlobSize + + ", detailed=" + + detailed + "]"; } @@ -701,11 +711,13 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("seed", seed); builder.field("blob_path", blobPath); - builder.startArray("blobs"); - for (BlobSpeedTestAction.Response blobResponse : blobResponses) { - blobResponse.toXContent(builder, params); + if (blobResponses.size() > 0) { + builder.startArray("details"); + for (BlobSpeedTestAction.Response blobResponse : blobResponses) { + blobResponse.toXContent(builder, params); + } + builder.endArray(); } - builder.endArray(); builder.field("listing_nanos", listingTimeNanos); builder.field("delete_nanos", deleteTimeNanos); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java index 1d4e0aed4d0a3..f9d86e597a9df 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java @@ -4,16 +4,14 @@ * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.rest.action.admin.cluster; +package org.elasticsearch.repositories.blobstore.testkit; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.repositories.blobstore.testkit.RepositorySpeedTestAction; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestStatusToXContentListener; -import java.io.IOException; import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.POST; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java index b899a44486fbb..60e5607cb9643 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java @@ -18,7 +18,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; -import org.elasticsearch.rest.action.admin.cluster.RestRepositorySpeedTestAction; import java.util.List; import java.util.function.Supplier; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java new file mode 100644 index 0000000000000..ebd62c1b3c502 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java @@ -0,0 +1,11 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +public class SpeedTestStatistics { + +} From 0e904d80ddcc2b7579fc7ab5f3b747947f493edd Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 11:48:47 +0000 Subject: [PATCH 06/43] Collect stats --- .../testkit/BlobSpeedTestAction.java | 47 ++++++- .../testkit/RepositorySpeedTestAction.java | 10 +- .../testkit/SpeedTestStatistics.java | 121 +++++++++++++++++- 3 files changed, 169 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java index f1ac99d45bc75..5d60dfa269049 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java @@ -65,7 +65,7 @@ * in this case we insist that the read succeeds. * * - * + * 
  *
  * +---------+                           +-------+                               +---------+
  * | Writer  |                           | Repo  |                               | Readers |
@@ -132,6 +132,7 @@
  *      |←-----------------                  |                                        |
  *      |                                    |                                        |
  *
+ *
* * * On success, details of how long everything took are returned. On failure, cancels the remote read tasks to try and avoid consuming @@ -799,6 +800,26 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); return builder; } + + long getWriteBytes() { + return blobLength + (overwrite ? blobLength : 0L); + } + + long getWriteThrottledNanos() { + return writeThrottledNanos; + } + + long getWriteElapsedNanos() { + return writeElapsedNanos + overwriteElapsedNanos; + } + + List getReadDetails() { + return readDetails; + } + + long getChecksumBytes() { + return checksumEnd - checksumStart; + } } public static class ReadDetail implements Writeable, ToXContentFragment { @@ -808,8 +829,8 @@ public static class ReadDetail implements Writeable, ToXContentFragment { private final boolean beforeWriteComplete; private final boolean isNotFound; private final long firstByteNanos; - private final long readNanos; private final long throttleNanos; + private final long elapsedNanos; public ReadDetail( String nodeId, @@ -817,7 +838,7 @@ public ReadDetail( boolean beforeWriteComplete, boolean isNotFound, long firstByteNanos, - long readNanos, + long elapsedNanos, long throttleNanos ) { this.nodeId = nodeId; @@ -825,8 +846,8 @@ public ReadDetail( this.beforeWriteComplete = beforeWriteComplete; this.isNotFound = isNotFound; this.firstByteNanos = firstByteNanos; - this.readNanos = readNanos; this.throttleNanos = throttleNanos; + this.elapsedNanos = elapsedNanos; } public ReadDetail(StreamInput in) throws IOException { @@ -835,8 +856,8 @@ public ReadDetail(StreamInput in) throws IOException { beforeWriteComplete = in.readBoolean(); isNotFound = in.readBoolean(); firstByteNanos = in.readVLong(); - readNanos = in.readVLong(); throttleNanos = in.readVLong(); + elapsedNanos = in.readVLong(); } @Override @@ -846,8 +867,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(beforeWriteComplete); out.writeBoolean(isNotFound); out.writeVLong(firstByteNanos); - out.writeVLong(readNanos); out.writeVLong(throttleNanos); + out.writeVLong(elapsedNanos); } @Override @@ -868,13 +889,25 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } else { builder.field("found", true); builder.field("first_byte_nanos", firstByteNanos); - builder.field("read_nanos", readNanos); + builder.field("read_nanos", elapsedNanos); builder.field("throttle_nanos", throttleNanos); } builder.endObject(); return builder; } + + long getFirstByteNanos() { + return firstByteNanos; + } + + long getThrottledNanos() { + return throttleNanos; + } + + long getElapsedNanos() { + return elapsedNanos; + } } } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java index 9746e6d227887..cfc6fa6bc9ce0 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java @@ -166,7 +166,7 @@ public static class AsyncAction { private final GroupedActionListener workersListener; private final Set expectedBlobs = ConcurrentCollections.newConcurrentSet(); private final List responses; - private final SpeedTestStatistics statistics = new SpeedTestStatistics(); + private final SpeedTestStatistics.Builder statistics = new SpeedTestStatistics.Builder(); @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private OptionalLong listingStartTimeNanos = OptionalLong.empty(); @@ -327,6 +327,7 @@ public void handleResponse(BlobSpeedTestAction.Response response) { responses.add(response); } } + statistics.add(response); processNextTask(); } @@ -463,6 +464,7 @@ private void sendResponse() { request.maxBlobSize, request.seed, blobPath, + statistics.build(), responses, deleteStartTimeNanos.getAsLong() - listingStartTimeNanos.getAsLong(), completionTimeNanos - deleteStartTimeNanos.getAsLong() @@ -644,6 +646,7 @@ public static class Response extends ActionResponse implements StatusToXContentO private final ByteSizeValue maxBlobSize; private final long seed; private final String blobPath; + private final SpeedTestStatistics statistics; private final List blobResponses; private final long listingTimeNanos; private final long deleteTimeNanos; @@ -655,6 +658,7 @@ public Response( ByteSizeValue maxBlobSize, long seed, String blobPath, + SpeedTestStatistics statistics, List blobResponses, long listingTimeNanos, long deleteTimeNanos @@ -665,6 +669,7 @@ public Response( this.maxBlobSize = maxBlobSize; this.seed = seed; this.blobPath = blobPath; + this.statistics = statistics; this.blobResponses = blobResponses; this.listingTimeNanos = listingTimeNanos; this.deleteTimeNanos = deleteTimeNanos; @@ -678,6 +683,7 @@ public Response(StreamInput in) throws IOException { maxBlobSize = new ByteSizeValue(in); seed = in.readLong(); blobPath = in.readString(); + statistics = new SpeedTestStatistics(in); blobResponses = in.readList(BlobSpeedTestAction.Response::new); listingTimeNanos = in.readVLong(); deleteTimeNanos = in.readVLong(); @@ -691,6 +697,7 @@ public void writeTo(StreamOutput out) throws IOException { maxBlobSize.writeTo(out); out.writeLong(seed); out.writeString(blobPath); + statistics.writeTo(out); out.writeList(blobResponses); out.writeVLong(listingTimeNanos); out.writeVLong(deleteTimeNanos); @@ -710,6 +717,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("max_blob_size", maxBlobSize); builder.field("seed", seed); builder.field("blob_path", blobPath); + builder.field("statistics", statistics); if (blobResponses.size() > 0) { builder.startArray("details"); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java index ebd62c1b3c502..2e7cfcf3c20ba 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java @@ -6,6 +6,125 @@ package org.elasticsearch.repositories.blobstore.testkit; -public class SpeedTestStatistics { +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import java.io.IOException; +import java.util.concurrent.atomic.LongAdder; + +public class SpeedTestStatistics implements Writeable, ToXContentFragment { + + private final long writeCount; + private final long writeBytes; + private final long writeThrottledNanos; + private final long writeElapsedNanos; + private final long readCount; + private final long readBytes; + private final long readWaitNanos; + private final long readThrottledNanos; + private final long readElapsedNanos; + + public SpeedTestStatistics(long writeCount, long writeBytes, long writeThrottledNanos, long writeElapsedNanos, + long readCount, long readBytes, long readWaitNanos, long readThrottledNanos, long readElapsedNanos) { + this.writeCount = writeCount; + this.writeBytes = writeBytes; + this.writeThrottledNanos = writeThrottledNanos; + this.writeElapsedNanos = writeElapsedNanos; + this.readCount = readCount; + this.readBytes = readBytes; + this.readWaitNanos = readWaitNanos; + this.readThrottledNanos = readThrottledNanos; + this.readElapsedNanos = readElapsedNanos; + } + + public SpeedTestStatistics(StreamInput in) throws IOException { + writeCount = in.readVLong(); + writeBytes = in.readVLong(); + writeThrottledNanos = in.readVLong(); + writeElapsedNanos = in.readVLong(); + readCount = in.readVLong(); + readBytes = in.readVLong(); + readWaitNanos = in.readVLong(); + readThrottledNanos = in.readVLong(); + readElapsedNanos = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(writeCount); + out.writeVLong(writeBytes); + out.writeVLong(writeThrottledNanos); + out.writeVLong(writeElapsedNanos); + out.writeVLong(readCount); + out.writeVLong(readBytes); + out.writeVLong(readWaitNanos); + out.writeVLong(readThrottledNanos); + out.writeVLong(readElapsedNanos); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject("write"); + builder.field("count", writeCount); + builder.field("total_bytes", writeBytes); + builder.field("throttled_nanos", writeThrottledNanos); + builder.field("elapsed_nanos", writeElapsedNanos); + builder.endObject(); + + builder.startObject("read"); + builder.field("count", readCount); + builder.field("total_bytes", readBytes); + builder.field("wait_nanos", readWaitNanos); + builder.field("throttled_nanos", readThrottledNanos); + builder.field("elapsed_nanos", readElapsedNanos); + builder.endObject(); + return builder; + } + + static class Builder { + + private final LongAdder writeCount = new LongAdder(); + private final LongAdder writeBytes = new LongAdder(); + private final LongAdder writeThrottledNanos = new LongAdder(); + private final LongAdder writeElapsedNanos = new LongAdder(); + + private final LongAdder readCount = new LongAdder(); + private final LongAdder readBytes = new LongAdder(); + private final LongAdder readWaitNanos = new LongAdder(); + private final LongAdder readThrottledNanos = new LongAdder(); + private final LongAdder readElapsedNanos = new LongAdder(); + + public SpeedTestStatistics build() { + return new SpeedTestStatistics( + writeCount.longValue(), + writeBytes.longValue(), + writeThrottledNanos.longValue(), + writeElapsedNanos.longValue(), + readCount.longValue(), + readBytes.longValue(), + readWaitNanos.longValue(), + readThrottledNanos.longValue(), + readElapsedNanos.longValue()); + } + + public void add(BlobSpeedTestAction.Response response) { + writeCount.add(1L); + writeBytes.add(response.getWriteBytes()); + writeThrottledNanos.add(response.getWriteThrottledNanos()); + writeElapsedNanos.add(response.getWriteElapsedNanos()); + + final long checksumBytes = response.getChecksumBytes(); + + for (final BlobSpeedTestAction.ReadDetail readDetail : response.getReadDetails()) { + readCount.add(1L); + readBytes.add(checksumBytes); + readWaitNanos.add(readDetail.getFirstByteNanos()); + readThrottledNanos.add(readDetail.getThrottledNanos()); + readElapsedNanos.add(readDetail.getElapsedNanos()); + } + } + } } From e32a3775d4dcd36d093f462855a10181228c9316 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 13:54:13 +0000 Subject: [PATCH 07/43] Rename to summary --- .../rest-api-spec/test/10_speed_test.yml | 18 +++++++++ .../testkit/RepositorySpeedTestAction.java | 18 ++++----- ...tStatistics.java => SpeedTestSummary.java} | 40 ++++++++++++------- 3 files changed, 52 insertions(+), 24 deletions(-) rename x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/{SpeedTestStatistics.java => SpeedTestSummary.java} (82%) diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml index 9e4f21d885005..88c33ece8b863 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml @@ -56,6 +56,15 @@ setup: - is_false: details - gte: { listing_nanos: 0} - gte: { delete_nanos: 0} + - eq: { summary.write.count: 10} + - gte: { summary.write.total_bytes: 0} + - gte: { summary.write.throttled_nanos: 0} + - gte: { summary.write.elapsed_nanos: 0} + - gte: { summary.read.count: 10} + - gte: { summary.read.total_bytes: 0} + - gte: { summary.read.wait_nanos: 0} + - gte: { summary.read.throttled_nanos: 0} + - gte: { summary.read.elapsed_nanos: 0} --- "Speed test with details": @@ -80,3 +89,12 @@ setup: - is_true: details - gte: { listing_nanos: 0} - gte: { delete_nanos: 0} + - eq: { summary.write.count: 10} + - gte: { summary.write.total_bytes: 0} + - gte: { summary.write.throttled_nanos: 0} + - gte: { summary.write.elapsed_nanos: 0} + - gte: { summary.read.count: 10} + - gte: { summary.read.total_bytes: 0} + - gte: { summary.read.wait_nanos: 0} + - gte: { summary.read.throttled_nanos: 0} + - gte: { summary.read.elapsed_nanos: 0} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java index cfc6fa6bc9ce0..9e72a3294e43c 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java @@ -166,7 +166,7 @@ public static class AsyncAction { private final GroupedActionListener workersListener; private final Set expectedBlobs = ConcurrentCollections.newConcurrentSet(); private final List responses; - private final SpeedTestStatistics.Builder statistics = new SpeedTestStatistics.Builder(); + private final SpeedTestSummary.Builder summary = new SpeedTestSummary.Builder(); @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private OptionalLong listingStartTimeNanos = OptionalLong.empty(); @@ -327,7 +327,7 @@ public void handleResponse(BlobSpeedTestAction.Response response) { responses.add(response); } } - statistics.add(response); + summary.add(response); processNextTask(); } @@ -464,7 +464,7 @@ private void sendResponse() { request.maxBlobSize, request.seed, blobPath, - statistics.build(), + summary.build(), responses, deleteStartTimeNanos.getAsLong() - listingStartTimeNanos.getAsLong(), completionTimeNanos - deleteStartTimeNanos.getAsLong() @@ -646,7 +646,7 @@ public static class Response extends ActionResponse implements StatusToXContentO private final ByteSizeValue maxBlobSize; private final long seed; private final String blobPath; - private final SpeedTestStatistics statistics; + private final SpeedTestSummary summary; private final List blobResponses; private final long listingTimeNanos; private final long deleteTimeNanos; @@ -658,7 +658,7 @@ public Response( ByteSizeValue maxBlobSize, long seed, String blobPath, - SpeedTestStatistics statistics, + SpeedTestSummary summary, List blobResponses, long listingTimeNanos, long deleteTimeNanos @@ -669,7 +669,7 @@ public Response( this.maxBlobSize = maxBlobSize; this.seed = seed; this.blobPath = blobPath; - this.statistics = statistics; + this.summary = summary; this.blobResponses = blobResponses; this.listingTimeNanos = listingTimeNanos; this.deleteTimeNanos = deleteTimeNanos; @@ -683,7 +683,7 @@ public Response(StreamInput in) throws IOException { maxBlobSize = new ByteSizeValue(in); seed = in.readLong(); blobPath = in.readString(); - statistics = new SpeedTestStatistics(in); + summary = new SpeedTestSummary(in); blobResponses = in.readList(BlobSpeedTestAction.Response::new); listingTimeNanos = in.readVLong(); deleteTimeNanos = in.readVLong(); @@ -697,7 +697,7 @@ public void writeTo(StreamOutput out) throws IOException { maxBlobSize.writeTo(out); out.writeLong(seed); out.writeString(blobPath); - statistics.writeTo(out); + summary.writeTo(out); out.writeList(blobResponses); out.writeVLong(listingTimeNanos); out.writeVLong(deleteTimeNanos); @@ -717,7 +717,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("max_blob_size", maxBlobSize); builder.field("seed", seed); builder.field("blob_path", blobPath); - builder.field("statistics", statistics); + builder.field("summary", summary); if (blobResponses.size() > 0) { builder.startArray("details"); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java similarity index 82% rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java index 2e7cfcf3c20ba..816cfe2df9912 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestStatistics.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java @@ -15,7 +15,7 @@ import java.io.IOException; import java.util.concurrent.atomic.LongAdder; -public class SpeedTestStatistics implements Writeable, ToXContentFragment { +public class SpeedTestSummary implements Writeable, ToXContentFragment { private final long writeCount; private final long writeBytes; @@ -27,8 +27,17 @@ public class SpeedTestStatistics implements Writeable, ToXContentFragment { private final long readThrottledNanos; private final long readElapsedNanos; - public SpeedTestStatistics(long writeCount, long writeBytes, long writeThrottledNanos, long writeElapsedNanos, - long readCount, long readBytes, long readWaitNanos, long readThrottledNanos, long readElapsedNanos) { + public SpeedTestSummary( + long writeCount, + long writeBytes, + long writeThrottledNanos, + long writeElapsedNanos, + long readCount, + long readBytes, + long readWaitNanos, + long readThrottledNanos, + long readElapsedNanos + ) { this.writeCount = writeCount; this.writeBytes = writeBytes; this.writeThrottledNanos = writeThrottledNanos; @@ -40,7 +49,7 @@ public SpeedTestStatistics(long writeCount, long writeBytes, long writeThrottled this.readElapsedNanos = readElapsedNanos; } - public SpeedTestStatistics(StreamInput in) throws IOException { + public SpeedTestSummary(StreamInput in) throws IOException { writeCount = in.readVLong(); writeBytes = in.readVLong(); writeThrottledNanos = in.readVLong(); @@ -97,17 +106,18 @@ static class Builder { private final LongAdder readThrottledNanos = new LongAdder(); private final LongAdder readElapsedNanos = new LongAdder(); - public SpeedTestStatistics build() { - return new SpeedTestStatistics( - writeCount.longValue(), - writeBytes.longValue(), - writeThrottledNanos.longValue(), - writeElapsedNanos.longValue(), - readCount.longValue(), - readBytes.longValue(), - readWaitNanos.longValue(), - readThrottledNanos.longValue(), - readElapsedNanos.longValue()); + public SpeedTestSummary build() { + return new SpeedTestSummary( + writeCount.longValue(), + writeBytes.longValue(), + writeThrottledNanos.longValue(), + writeElapsedNanos.longValue(), + readCount.longValue(), + readBytes.longValue(), + readWaitNanos.longValue(), + readThrottledNanos.longValue(), + readElapsedNanos.longValue() + ); } public void add(BlobSpeedTestAction.Response response) { From c237aa505ff2fb1d0bfbd11cd5878d59bea574b4 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 14:49:25 +0000 Subject: [PATCH 08/43] Fix RandomBlobContentBytesReference --- .../RandomBlobContentBytesReference.java | 3 ++ .../blobstore/testkit/SpeedTestSummary.java | 4 ++ .../RandomBlobContentBytesReferenceTests.java | 51 +++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java index d6bae5524d273..f856431b5134d 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java @@ -69,6 +69,9 @@ public BytesRefIterator iterator() { @Override public BytesRef next() { final int block = nextBlock++; + if (block > lastBlock) { + return null; + } randomBlobContent.ensureNotCancelled(block * buffer.length + "/" + length); final int end; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java index 816cfe2df9912..09d50b6391543 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java @@ -76,6 +76,8 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startObject("write"); builder.field("count", writeCount); builder.field("total_bytes", writeBytes); @@ -89,6 +91,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("wait_nanos", readWaitNanos); builder.field("throttled_nanos", readThrottledNanos); builder.field("elapsed_nanos", readElapsedNanos); + builder.endObject(); + builder.endObject(); return builder; } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java new file mode 100644 index 0000000000000..d8739417e9435 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.zip.CRC32; + +import static org.elasticsearch.repositories.blobstore.testkit.RandomBlobContent.BUFFER_SIZE; +import static org.hamcrest.Matchers.equalTo; + +public class RandomBlobContentBytesReferenceTests extends ESTestCase { + + public void testStreamInput() throws IOException { + final AtomicBoolean readComplete = new AtomicBoolean(); + final RandomBlobContent randomBlobContent = new RandomBlobContent( + "repo", + randomLong(), + () -> false, + () -> assertTrue("multiple notifications", readComplete.compareAndSet(false, true)) + ); + + final int length = randomSize(); + final int checksumStart = between(0, length - 1); + final int checksumEnd = between(checksumStart, length); + + final byte[] output = new RandomBlobContentBytesReference(randomBlobContent, length).streamInput().readAllBytes(); + + assertThat(output.length, equalTo(length)); + assertTrue(readComplete.get()); + + for (int i = BUFFER_SIZE; i < output.length; i++) { + assertThat("output repeats at position " + i, output[i], equalTo(output[i % BUFFER_SIZE])); + } + + final CRC32 crc32 = new CRC32(); + crc32.update(output, checksumStart, checksumEnd - checksumStart); + assertThat("checksum computed correctly", randomBlobContent.getChecksum(checksumStart, checksumEnd), equalTo(crc32.getValue())); + } + + private static int randomSize() { + return between(0, 30000); + } + +} From 36a6e52f59425036ddb5dd97d2d08bfa1d206159 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 14:56:09 +0000 Subject: [PATCH 09/43] Fixes --- .../test/resources/rest-api-spec/test/10_speed_test.yml | 4 ++-- .../testkit/RandomBlobContentBytesReferenceTests.java | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml index 88c33ece8b863..6088e1847e1ff 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml @@ -56,7 +56,7 @@ setup: - is_false: details - gte: { listing_nanos: 0} - gte: { delete_nanos: 0} - - eq: { summary.write.count: 10} + - match: { summary.write.count: 10} - gte: { summary.write.total_bytes: 0} - gte: { summary.write.throttled_nanos: 0} - gte: { summary.write.elapsed_nanos: 0} @@ -89,7 +89,7 @@ setup: - is_true: details - gte: { listing_nanos: 0} - gte: { delete_nanos: 0} - - eq: { summary.write.count: 10} + - match: { summary.write.count: 10} - gte: { summary.write.total_bytes: 0} - gte: { summary.write.throttled_nanos: 0} - gte: { summary.write.elapsed_nanos: 0} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java index d8739417e9435..70c88a357967f 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java @@ -20,10 +20,10 @@ public class RandomBlobContentBytesReferenceTests extends ESTestCase { public void testStreamInput() throws IOException { final AtomicBoolean readComplete = new AtomicBoolean(); final RandomBlobContent randomBlobContent = new RandomBlobContent( - "repo", - randomLong(), - () -> false, - () -> assertTrue("multiple notifications", readComplete.compareAndSet(false, true)) + "repo", + randomLong(), + () -> false, + () -> assertTrue("multiple notifications", readComplete.compareAndSet(false, true)) ); final int length = randomSize(); From 280f982f74aa505a02b790acc3a31fc855c9fb7b Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 16:02:30 +0000 Subject: [PATCH 10/43] Mark speed test actions as operator-only --- .../xpack/security/operator/OperatorOnlyRegistry.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java index b3b633199d8f8..a61d5c736887b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java @@ -24,7 +24,12 @@ public class OperatorOnlyRegistry { "cluster:admin/autoscaling/put_autoscaling_policy", "cluster:admin/autoscaling/delete_autoscaling_policy", "cluster:admin/autoscaling/get_autoscaling_policy", - "cluster:admin/autoscaling/get_autoscaling_capacity"); + "cluster:admin/autoscaling/get_autoscaling_capacity", + // Repository speed test actions are not mentioned in core, literal strings are needed. + "cluster:admin/repository/speed_test", + "cluster:admin/repository/speed_test/blob", + "cluster:admin/repository/speed_test/blob/read" + ); /** * Check whether the given action and request qualify as operator-only. The method returns From 64c220d42344fa68a74afb05261a2efcfe82f4e7 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 16:19:42 +0000 Subject: [PATCH 11/43] Report blob-level request on all failures --- .../testkit/BlobSpeedTestAction.java | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java index 5d60dfa269049..7eeb0399750ea 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java @@ -281,7 +281,9 @@ private void writeRandomBlob(boolean atomic, boolean failIfExists, Runnable onLa ); final AtomicLong throttledNanos = new AtomicLong(); - logger.trace("writing blob [atomic={}, failIfExists={}] for [{}]", atomic, failIfExists, request.getDescription()); + if (logger.isTraceEnabled()) { + logger.trace("writing blob [atomic={}, failIfExists={}] for [{}]", atomic, failIfExists, request.getDescription()); + } final long startNanos = System.nanoTime(); ActionListener.completeWith(stepListener, () -> { if (atomic || (request.targetLength <= Integer.MAX_VALUE && random.nextBoolean())) { @@ -314,20 +316,26 @@ public StreamInput streamInput() throws IOException { } final long elapsedNanos = System.nanoTime() - startNanos; final long checksum = content.getChecksum(checksumStart, checksumEnd); - logger.trace("finished writing blob for [{}], got checksum [{}]", request.getDescription(), checksum); + if (logger.isTraceEnabled()) { + logger.trace("finished writing blob for [{}], got checksum [{}]", request.getDescription(), checksum); + } return new WriteDetails(request.targetLength, elapsedNanos, throttledNanos.get(), checksum); }); } private void doReadBeforeWriteComplete() { if (earlyReadNodes.isEmpty() == false) { - logger.trace("sending read request to [{}] for [{}] before write complete", earlyReadNodes, request.getDescription()); + if (logger.isTraceEnabled()) { + logger.trace("sending read request to [{}] for [{}] before write complete", earlyReadNodes, request.getDescription()); + } readOnNodes(earlyReadNodes, true); } } private void doReadAfterWrite() { - logger.trace("sending read request to [{}] for [{}] after write complete", readNodes, request.getDescription()); + if (logger.isTraceEnabled()) { + logger.trace("sending read request to [{}] for [{}] after write complete", readNodes, request.getDescription()); + } readOnNodes(readNodes, false); } @@ -365,8 +373,6 @@ public void onFailure(Exception e) { + (beforeWriteComplete ? "before" : "after") + " write complete) failed on node [" + node - + "] as part of [" - + request.getDescription() + "]", e ) @@ -406,7 +412,9 @@ private void cancelReadsCleanUpAndReturnFailure(Exception exception) { } private void cleanUpAndReturnFailure(Exception exception) { - logger.trace(new ParameterizedMessage("speed test failed [{}] cleaning up", request.getDescription()), exception); + if (logger.isTraceEnabled()) { + logger.trace(new ParameterizedMessage("speed test failed [{}] cleaning up", request.getDescription()), exception); + } try { blobContainer.deleteBlobsIgnoringIfNotExists(List.of(request.blobName)); } catch (IOException ioException) { @@ -421,16 +429,19 @@ private void cleanUpAndReturnFailure(Exception exception) { exception ); } - listener.onFailure(exception); + listener.onFailure( + new RepositoryVerificationException( + request.getRepositoryName(), + "failure processing [" + request.getDescription() + "]", + exception + ) + ); } private void onReadsComplete(Collection responses, WriteDetails write1Details, @Nullable WriteDetails write2Details) { if (task.isCancelled()) { cleanUpAndReturnFailure( - new RepositoryVerificationException( - request.getRepositoryName(), - "cancelled [" + request.getDescription() + "] during checksum verification" - ) + new RepositoryVerificationException(request.getRepositoryName(), "cancelled during checksum verification") ); return; } @@ -458,11 +469,7 @@ private void onReadsComplete(Collection responses, WriteDetails wr } else { nodeFailure = new RepositoryVerificationException( request.getRepositoryName(), - "node [" - + nodeResponse.node - + "] reported blob not found after it was written [" - + request.getDescription() - + "]" + "node [" + nodeResponse.node + "] reported blob not found after it was written" ); } } else { @@ -484,8 +491,6 @@ private void onReadsComplete(Collection responses, WriteDetails wr + expectedChecksumDescription + "] but read [" + response - + "] as part of [" - + request.getDescription() + "]" ); } From 29f5d501c4c070844a19f4952d5b0da00a4c447e Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 18:20:45 +0000 Subject: [PATCH 12/43] Reroute to arbitrary snapshot node, not the master --- .../apis/repo-speed-test-api.asciidoc | 5 - .../repositories/RepositoriesService.java | 2 +- .../api/snapshot.repository_speed_test.json | 4 - .../rest-api-spec/test/10_speed_test.yml | 4 + .../testkit/RepositorySpeedTestAction.java | 166 ++++++++++++------ .../RestRepositorySpeedTestAction.java | 1 - 6 files changed, 121 insertions(+), 61 deletions(-) diff --git a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc index 137aeab151870..a7246e54b6b25 100644 --- a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc @@ -115,11 +115,6 @@ experiment. during the test. Defaults to `10mb`. For realistic experiments you should set this to at least `2gb`. -`master_timeout`:: -(Optional, <>) Specifies the period of time to wait for -a connection to the master node. If no response is received before the timeout -expires, the request fails and returns an error. Defaults to `30s`. - `timeout`:: (Optional, <>) Specifies the period of time to wait for the test to complete. If no response is received before the timeout expires, diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 2a6e49645e6e2..eef91f1f4ac39 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -284,7 +284,7 @@ protected void doRun() { }); } - static boolean isDedicatedVotingOnlyNode(Set roles) { + public static boolean isDedicatedVotingOnlyNode(Set roles) { return roles.contains(DiscoveryNodeRole.MASTER_ROLE) && roles.contains(DiscoveryNodeRole.DATA_ROLE) == false && roles.stream().anyMatch(role -> role.roleName().equals("voting_only")); } diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json index 01db6c20e4d38..c8dee6e3d7e21 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json @@ -43,10 +43,6 @@ "type":"time", "description":"Explicit operation timeout" }, - "master_timeout":{ - "type":"time", - "description":"Explicit operation timeout for connection to master node" - }, "detailed":{ "type":"boolean", "description":"Whether to return detailed results or a summary. Defaults to `false`." diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml index 6088e1847e1ff..8cf08873d72e5 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml @@ -47,6 +47,8 @@ setup: concurrency: 5 max_blob_size: 1mb + - is_true: coordinating_node.id + - is_true: coordinating_node.name - match: { repository: test_repo } - match: { blob_count: 10 } - match: { concurrency: 5 } @@ -80,6 +82,8 @@ setup: max_blob_size: 1mb detailed: true + - is_true: coordinating_node.id + - is_true: coordinating_node.name - match: { repository: test_repo } - match: { blob_count: 10 } - match: { concurrency: 5 } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java index 9e72a3294e43c..354a4ead838b4 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java @@ -13,17 +13,16 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionListenerResponseHandler; +import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.GroupedActionListener; +import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.action.support.ThreadedActionListener; -import org.elasticsearch.action.support.master.MasterNodeRequest; -import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.block.ClusterBlockException; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; @@ -85,64 +84,106 @@ private RepositorySpeedTestAction() { super(NAME, Response::new); } - public static class TransportAction extends TransportMasterNodeAction { - // Needs to run on a non-voting-only master-eligible node or a data node so that it has access to the repository so it can clean up - // at the end. However today there's no good way to reroute an action onto a suitable node apart from routing it tothe elected - // master, so we run it there. This should be no big deal, coordinating the test isn't particularly heavy. + public static class TransportAction extends HandledTransportAction { + private final TransportService transportService; + private final ClusterService clusterService; private final RepositoriesService repositoriesService; @Inject public TransportAction( TransportService transportService, - ClusterService clusterService, - ThreadPool threadPool, ActionFilters actionFilters, - RepositoriesService repositoriesService, - IndexNameExpressionResolver indexNameExpressionResolver + ClusterService clusterService, + RepositoriesService repositoriesService ) { - super( - NAME, - transportService, - clusterService, - threadPool, - actionFilters, - RepositorySpeedTestAction.Request::new, - indexNameExpressionResolver, - RepositorySpeedTestAction.Response::new, - ThreadPool.Names.SAME - ); + super(NAME, transportService, actionFilters, RepositorySpeedTestAction.Request::new, ThreadPool.Names.SAME); + this.transportService = transportService; + this.clusterService = clusterService; this.repositoriesService = repositoriesService; } @Override - protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) { - final Repository repository = repositoriesService.repository(request.getRepositoryName()); - if (repository instanceof BlobStoreRepository == false) { - throw new IllegalArgumentException("repository [" + request.getRepositoryName() + "] is not a blob-store repository"); - } - if (repository.isReadOnly()) { - throw new IllegalArgumentException("repository [" + request.getRepositoryName() + "] is read-only"); - } + protected void doExecute(Task task, Request request, ActionListener listener) { + final ClusterState state = clusterService.state(); + final ThreadPool threadPool = transportService.getThreadPool(); request.reseed(threadPool.relativeTimeInMillis()); - assert task instanceof CancellableTask; - new AsyncAction( - transportService, - (BlobStoreRepository) repository, - (CancellableTask) task, - request, - state.nodes(), - threadPool::relativeTimeInMillis, - listener - ).run(); + final DiscoveryNode localNode = transportService.getLocalNode(); + if (isSnapshotNode(localNode)) { + final Repository repository = repositoriesService.repository(request.getRepositoryName()); + if (repository instanceof BlobStoreRepository == false) { + throw new IllegalArgumentException("repository [" + request.getRepositoryName() + "] is not a blob-store repository"); + } + if (repository.isReadOnly()) { + throw new IllegalArgumentException("repository [" + request.getRepositoryName() + "] is read-only"); + } + + assert task instanceof CancellableTask; + new AsyncAction( + transportService, + (BlobStoreRepository) repository, + (CancellableTask) task, + request, + state.nodes(), + threadPool::relativeTimeInMillis, + listener + ).run(); + return; + } + + if (request.getReroutedFrom() != null) { + assert false : request.getReroutedFrom(); + throw new IllegalArgumentException( + "speed test of repository [" + + request.getRepositoryName() + + "] rerouted from [" + + request.getReroutedFrom() + + "] to non-snapshot node" + ); + } + + request.reroutedFrom(localNode); + final List snapshotNodes = getSnapshotNodes(state.nodes()); + if (snapshotNodes.isEmpty()) { + listener.onFailure( + new IllegalArgumentException( + "no snapshot nodes found for speed test of repository [" + request.getRepositoryName() + "]" + ) + ); + } else { + if (snapshotNodes.size() > 1) { + snapshotNodes.remove(state.nodes().getMasterNode()); + } + final DiscoveryNode targetNode = snapshotNodes.get(new Random(request.getSeed()).nextInt(snapshotNodes.size())); + RepositorySpeedTestAction.logger.trace("rerouting speed test [{}] to [{}]", request.getDescription(), targetNode); + transportService.sendChildRequest( + targetNode, + NAME, + request, + task, + TransportRequestOptions.EMPTY, + new ActionListenerResponseHandler<>(listener, Response::new) + ); + } } + } - @Override - protected ClusterBlockException checkBlock(Request request, ClusterState state) { - return null; + private static boolean isSnapshotNode(DiscoveryNode discoveryNode) { + return (discoveryNode.isDataNode() || discoveryNode.isMasterNode()) + && RepositoriesService.isDedicatedVotingOnlyNode(discoveryNode.getRoles()) == false; + } + + private static List getSnapshotNodes(DiscoveryNodes discoveryNodes) { + final ObjectContainer nodesContainer = discoveryNodes.getMasterAndDataNodes().values(); + final List nodes = new ArrayList<>(nodesContainer.size()); + for (ObjectCursor cursor : nodesContainer) { + if (isSnapshotNode(cursor.value)) { + nodes.add(cursor.value); + } } + return nodes; } public static class AsyncAction { @@ -263,11 +304,7 @@ public void run() { final Random random = new Random(request.getSeed()); - final ObjectContainer nodesContainer = discoveryNodes.getMasterAndDataNodes().values(); - final List nodes = new ArrayList<>(nodesContainer.size()); - for (ObjectCursor cursor : nodesContainer) { - nodes.add(cursor.value); - } + final List nodes = getSnapshotNodes(discoveryNodes); final long maxBlobSize = request.getMaxBlobSize().getBytes(); final List blobSizes = new ArrayList<>(); @@ -458,6 +495,8 @@ private void sendResponse() { listener.onResponse( new Response( + transportService.getLocalNode().getId(), + transportService.getLocalNode().getName(), request.getRepositoryName(), request.blobCount, request.concurrency, @@ -498,7 +537,7 @@ public String toString() { } } - public static class Request extends MasterNodeRequest { + public static class Request extends ActionRequest { private final String repositoryName; @@ -508,6 +547,7 @@ public static class Request extends MasterNodeRequest { private TimeValue timeout = TimeValue.timeValueSeconds(30); private ByteSizeValue maxBlobSize = ByteSizeValue.ofMb(10); private boolean detailed = false; + private DiscoveryNode reroutedFrom = null; public Request(String repositoryName) { this.repositoryName = repositoryName; @@ -522,6 +562,7 @@ public Request(StreamInput in) throws IOException { timeout = in.readTimeValue(); maxBlobSize = new ByteSizeValue(in); detailed = in.readBoolean(); + reroutedFrom = in.readOptionalWriteable(DiscoveryNode::new); } @Override @@ -539,6 +580,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeTimeValue(timeout); maxBlobSize.writeTo(out); out.writeBoolean(detailed); + out.writeOptionalWriteable(reroutedFrom); } @Override @@ -607,6 +649,14 @@ public boolean getDetailed() { return detailed; } + public DiscoveryNode getReroutedFrom() { + return reroutedFrom; + } + + public void reroutedFrom(DiscoveryNode discoveryNode) { + reroutedFrom = discoveryNode; + } + @Override public String toString() { return "Request{" + getDescription() + '}'; @@ -640,6 +690,8 @@ public void reseed(long newSeed) { public static class Response extends ActionResponse implements StatusToXContentObject { + private final String coordinatingNodeId; + private final String coordinatingNodeName; private final String repositoryName; private final int blobCount; private final int concurrency; @@ -652,6 +704,8 @@ public static class Response extends ActionResponse implements StatusToXContentO private final long deleteTimeNanos; public Response( + String coordinatingNodeId, + String coordinatingNodeName, String repositoryName, int blobCount, int concurrency, @@ -663,6 +717,8 @@ public Response( long listingTimeNanos, long deleteTimeNanos ) { + this.coordinatingNodeId = coordinatingNodeId; + this.coordinatingNodeName = coordinatingNodeName; this.repositoryName = repositoryName; this.blobCount = blobCount; this.concurrency = concurrency; @@ -677,6 +733,8 @@ public Response( public Response(StreamInput in) throws IOException { super(in); + coordinatingNodeId = in.readString(); + coordinatingNodeName = in.readString(); repositoryName = in.readString(); blobCount = in.readVInt(); concurrency = in.readVInt(); @@ -691,6 +749,8 @@ public Response(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { + out.writeString(coordinatingNodeId); + out.writeString(coordinatingNodeName); out.writeString(repositoryName); out.writeVInt(blobCount); out.writeVInt(concurrency); @@ -711,6 +771,12 @@ public RestStatus status() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + + builder.startObject("coordinating_node"); + builder.field("id", coordinatingNodeId); + builder.field("name", coordinatingNodeName); + builder.endObject(); + builder.field("repository", repositoryName); builder.field("blob_count", blobCount); builder.field("concurrency", concurrency); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java index f9d86e597a9df..39b440e2f9509 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java @@ -37,7 +37,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC verifyRepositoryRequest.seed(request.paramAsLong("seed", verifyRepositoryRequest.getSeed())); verifyRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", verifyRepositoryRequest.getMaxBlobSize())); verifyRepositoryRequest.timeout(request.paramAsTime("timeout", verifyRepositoryRequest.getTimeout())); - verifyRepositoryRequest.masterNodeTimeout(request.paramAsTime("master_timeout", verifyRepositoryRequest.masterNodeTimeout())); verifyRepositoryRequest.detailed(request.paramAsBoolean("detailed", verifyRepositoryRequest.getDetailed())); RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); From 75705ec53fabc873e698be81b6ff9fd263410a6e Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 15 Jan 2021 19:40:26 +0000 Subject: [PATCH 13/43] Add TODOs so the github comments don't get lost --- .../snapshot-restore/apis/repo-speed-test-api.asciidoc | 5 +++++ .../xpack/security/operator/OperatorOnlyRegistry.java | 1 + x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle | 1 + .../blobstore/testkit/RepositorySpeedTestIT.java | 3 +++ .../blobstore/testkit/RepositorySpeedTestAction.java | 1 + 5 files changed, 11 insertions(+) diff --git a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc index a7246e54b6b25..fa65a49bc602f 100644 --- a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc @@ -4,6 +4,11 @@ ++++ Repository speed test ++++ +// TODO TBD I originally styled this as an "extended verification API" but feel +// that verification is too strong a word. We certainly report errors if it +// finds any, but no errors doesn't mean the repository is 100% correct, nor +// does it meet any performance threshold. A speed test implies weaker claims +// so I expect it would be less misleading. Measures the performance characteristics of a snapshot repository. diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java index a61d5c736887b..07a6e0524f78d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java @@ -26,6 +26,7 @@ public class OperatorOnlyRegistry { "cluster:admin/autoscaling/get_autoscaling_policy", "cluster:admin/autoscaling/get_autoscaling_capacity", // Repository speed test actions are not mentioned in core, literal strings are needed. + // TODO consider whether these are operator-only actions "cluster:admin/repository/speed_test", "cluster:admin/repository/speed_test/blob", "cluster:admin/repository/speed_test/blob/read" diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle index 1fdadcfa0b1c0..fe8a120231301 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/build.gradle @@ -7,6 +7,7 @@ dependencies { testImplementation project(path: xpackModule('snapshot-repo-test-kit'), configuration: 'testArtifacts') } +// TODO we want 3rd-party tests for other repository types too final File repoDir = file("$buildDir/testclusters/repo") tasks.named("integTest").configure { diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java index 0d5a7c9db34d4..359c98f263a34 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java @@ -51,6 +51,9 @@ public class RepositorySpeedTestIT extends AbstractSnapshotIntegTestCase { + // These tests are incomplete and do not currently verify that the speed test picks up on any particular repository-side failures + // TODO complete these tests + @Before public void suppressConsistencyChecks() { disableRepoConsistencyCheck("repository is not used for snapshots"); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java index 354a4ead838b4..21087088c4c93 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java @@ -438,6 +438,7 @@ private void tryListAndCleanup(final int retryCount) { retry = false; deleteContainerAndSendResponse(); } else { + // TODO do we need this retry mechanism any more? S3 used not to have consistent listings but it might be ok now. logger.debug( "expected blobs [{}] missing in [{}:{}], trying again; retry count = {}", request.repositoryName, From 71800455cdd290deaa2aa549d29c8fd61de7690e Mon Sep 17 00:00:00 2001 From: David Turner Date: Sat, 16 Jan 2021 10:31:27 +0000 Subject: [PATCH 14/43] Expose magic read-node-count parameters --- .../apis/repo-speed-test-api.asciidoc | 9 ++++ .../api/snapshot.repository_speed_test.json | 8 +++ .../rest-api-spec/test/10_speed_test.yml | 6 +++ .../testkit/BlobSpeedTestAction.java | 20 ++++++-- .../testkit/RepositorySpeedTestAction.java | 49 +++++++++++++++++++ .../RestRepositorySpeedTestAction.java | 4 ++ 6 files changed, 91 insertions(+), 5 deletions(-) diff --git a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc index fa65a49bc602f..ff1233d7d45b3 100644 --- a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc @@ -109,6 +109,15 @@ at least `2000`. (Optional, integer) The number of write operations to perform concurrently. Defaults to `10`. +`read_node_count`:: +(Optional, integer) The number of nodes on which to perform a read operation +after writing each blob. Defaults to `10`. + +`early_read_node_count`:: +(Optional, integer) The number of nodes on which to perform an early read +operation while writing each blob. Defaults to `2`. Early read operations are +only rarely performed. + `seed`:: (Optional, integer) The seed for the pseudo-random number generator used to generate the list of operations performed during the test. To repeat the same diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json index c8dee6e3d7e21..040bee4eaeaf7 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json @@ -31,6 +31,14 @@ "type":"number", "description":"Number of operations to run concurrently during the test" }, + "read_node_count":{ + "type":"number", + "description":"Number of nodes on which to read a blob after writing" + }, + "early_read_node_count":{ + "type":"number", + "description":"Number of nodes on which to read a blob during writing" + }, "seed":{ "type":"number", "description":"Seed for the random number generator used to create the test workload" diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml index 8cf08873d72e5..8b84b38ea9b7f 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml @@ -46,12 +46,16 @@ setup: blob_count: 10 concurrency: 5 max_blob_size: 1mb + read_node_count: 2 + early_read_node_count: 1 - is_true: coordinating_node.id - is_true: coordinating_node.name - match: { repository: test_repo } - match: { blob_count: 10 } - match: { concurrency: 5 } + - match: { read_node_count: 2 } + - match: { early_read_node_count: 1 } - match: { max_blob_size: 1mb } - is_true: seed - is_true: blob_path @@ -87,6 +91,8 @@ setup: - match: { repository: test_repo } - match: { blob_count: 10 } - match: { concurrency: 5 } + - match: { read_node_count: 10 } + - match: { early_read_node_count: 2 } - match: { max_blob_size: 1mb } - is_true: seed - is_true: blob_path diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java index 7eeb0399750ea..02acb25615cf1 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java @@ -222,21 +222,21 @@ static class BlobSpeedTest { checksumWholeBlob = random.nextBoolean(); if (checksumWholeBlob) { checksumStart = 0L; - checksumEnd = this.request.targetLength; + checksumEnd = request.targetLength; } else { - checksumStart = randomLongBetween(0L, this.request.targetLength); - checksumEnd = randomLongBetween(checksumStart + 1, this.request.targetLength + 1); + checksumStart = randomLongBetween(0L, request.targetLength); + checksumEnd = randomLongBetween(checksumStart + 1, request.targetLength + 1); } final ArrayList nodes = new ArrayList<>(request.nodes); // copy for shuffling purposes if (request.readEarly) { Collections.shuffle(nodes, random); - earlyReadNodes = nodes.stream().limit(2).collect(Collectors.toList()); // TODO magic 2 + earlyReadNodes = nodes.stream().limit(request.earlyReadNodeCount).collect(Collectors.toList()); } else { earlyReadNodes = List.of(); } Collections.shuffle(nodes, random); - readNodes = nodes.stream().limit(10).collect(Collectors.toList()); // TODO magic 10 + readNodes = nodes.stream().limit(request.readNodeCount).collect(Collectors.toList()); final StepListener> readsCompleteStep = new StepListener<>(); readNodesListener = new GroupedActionListener<>( @@ -585,6 +585,8 @@ public static class Request extends ActionRequest implements TaskAwareRequest { private final long targetLength; private final long seed; private final List nodes; + private final int readNodeCount; + private final int earlyReadNodeCount; private final boolean readEarly; private final boolean writeAndOverwrite; @@ -595,6 +597,8 @@ public static class Request extends ActionRequest implements TaskAwareRequest { long targetLength, long seed, List nodes, + int readNodeCount, + int earlyReadNodeCount, boolean readEarly, boolean writeAndOverwrite ) { @@ -606,6 +610,8 @@ public static class Request extends ActionRequest implements TaskAwareRequest { this.targetLength = targetLength; this.seed = seed; this.nodes = nodes; + this.readNodeCount = readNodeCount; + this.earlyReadNodeCount = earlyReadNodeCount; this.readEarly = readEarly; this.writeAndOverwrite = writeAndOverwrite; } @@ -618,6 +624,8 @@ public static class Request extends ActionRequest implements TaskAwareRequest { targetLength = in.readVLong(); seed = in.readLong(); nodes = in.readList(DiscoveryNode::new); + readNodeCount = in.readVInt(); + earlyReadNodeCount = in.readVInt(); readEarly = in.readBoolean(); writeAndOverwrite = in.readBoolean(); } @@ -631,6 +639,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(targetLength); out.writeLong(seed); out.writeList(nodes); + out.writeVInt(readNodeCount); + out.writeVInt(earlyReadNodeCount); out.writeBoolean(readEarly); out.writeBoolean(writeAndOverwrite); } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java index 21087088c4c93..b6b99d13acef9 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java @@ -325,6 +325,8 @@ public void run() { targetLength, random.nextLong(), nodes, + request.readNodeCount, + request.earlyReadNodeCount, smallBlob && (random.nextInt(50) == 0), // TODO magic 50 repository.supportURLRepo() && smallBlob && random.nextInt(50) == 0 // TODO magic 50 ) @@ -501,6 +503,8 @@ private void sendResponse() { request.getRepositoryName(), request.blobCount, request.concurrency, + request.readNodeCount, + request.earlyReadNodeCount, request.maxBlobSize, request.seed, blobPath, @@ -544,6 +548,8 @@ public static class Request extends ActionRequest { private int blobCount = 100; private int concurrency = 10; + private int readNodeCount = 10; + private int earlyReadNodeCount = 2; private long seed = 0L; private TimeValue timeout = TimeValue.timeValueSeconds(30); private ByteSizeValue maxBlobSize = ByteSizeValue.ofMb(10); @@ -560,6 +566,8 @@ public Request(StreamInput in) throws IOException { seed = in.readLong(); blobCount = in.readVInt(); concurrency = in.readVInt(); + readNodeCount = in.readVInt(); + earlyReadNodeCount = in.readVInt(); timeout = in.readTimeValue(); maxBlobSize = new ByteSizeValue(in); detailed = in.readBoolean(); @@ -578,6 +586,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeLong(seed); out.writeVInt(blobCount); out.writeVInt(concurrency); + out.writeVInt(readNodeCount); + out.writeVInt(earlyReadNodeCount); out.writeTimeValue(timeout); maxBlobSize.writeTo(out); out.writeBoolean(detailed); @@ -658,6 +668,28 @@ public void reroutedFrom(DiscoveryNode discoveryNode) { reroutedFrom = discoveryNode; } + public void readNodeCount(int readNodeCount) { + if (readNodeCount <= 0) { + throw new IllegalArgumentException("readNodeCount must be >0, but was [" + readNodeCount + "]"); + } + this.readNodeCount = readNodeCount; + } + + public int getReadNodeCount() { + return readNodeCount; + } + + public void earlyReadNodeCount(int earlyReadNodeCount) { + if (earlyReadNodeCount <= 0) { + throw new IllegalArgumentException("earlyReadNodeCount must be >0, but was [" + earlyReadNodeCount + "]"); + } + this.earlyReadNodeCount = earlyReadNodeCount; + } + + public int getEarlyReadNodeCount() { + return earlyReadNodeCount; + } + @Override public String toString() { return "Request{" + getDescription() + '}'; @@ -671,6 +703,10 @@ public String getDescription() { + blobCount + ", concurrency=" + concurrency + + ", readNodeCount=" + + readNodeCount + + ", earlyReadNodeCount=" + + earlyReadNodeCount + ", seed=" + seed + ", timeout=" @@ -687,6 +723,7 @@ public void reseed(long newSeed) { seed = newSeed; } } + } public static class Response extends ActionResponse implements StatusToXContentObject { @@ -696,6 +733,8 @@ public static class Response extends ActionResponse implements StatusToXContentO private final String repositoryName; private final int blobCount; private final int concurrency; + private final int readNodeCount; + private final int earlyReadNodeCount; private final ByteSizeValue maxBlobSize; private final long seed; private final String blobPath; @@ -710,6 +749,8 @@ public Response( String repositoryName, int blobCount, int concurrency, + int readNodeCount, + int earlyReadNodeCount, ByteSizeValue maxBlobSize, long seed, String blobPath, @@ -723,6 +764,8 @@ public Response( this.repositoryName = repositoryName; this.blobCount = blobCount; this.concurrency = concurrency; + this.readNodeCount = readNodeCount; + this.earlyReadNodeCount = earlyReadNodeCount; this.maxBlobSize = maxBlobSize; this.seed = seed; this.blobPath = blobPath; @@ -739,6 +782,8 @@ public Response(StreamInput in) throws IOException { repositoryName = in.readString(); blobCount = in.readVInt(); concurrency = in.readVInt(); + readNodeCount = in.readVInt(); + earlyReadNodeCount = in.readVInt(); maxBlobSize = new ByteSizeValue(in); seed = in.readLong(); blobPath = in.readString(); @@ -755,6 +800,8 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(repositoryName); out.writeVInt(blobCount); out.writeVInt(concurrency); + out.writeVInt(readNodeCount); + out.writeVInt(earlyReadNodeCount); maxBlobSize.writeTo(out); out.writeLong(seed); out.writeString(blobPath); @@ -781,6 +828,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("repository", repositoryName); builder.field("blob_count", blobCount); builder.field("concurrency", concurrency); + builder.field("read_node_count", readNodeCount); + builder.field("early_read_node_count", earlyReadNodeCount); builder.field("max_blob_size", maxBlobSize); builder.field("seed", seed); builder.field("blob_path", blobPath); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java index 39b440e2f9509..7c1e5f7f42804 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java @@ -34,6 +34,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC verifyRepositoryRequest.blobCount(request.paramAsInt("blob_count", verifyRepositoryRequest.getBlobCount())); verifyRepositoryRequest.concurrency(request.paramAsInt("concurrency", verifyRepositoryRequest.getConcurrency())); + verifyRepositoryRequest.readNodeCount(request.paramAsInt("read_node_count", verifyRepositoryRequest.getReadNodeCount())); + verifyRepositoryRequest.earlyReadNodeCount( + request.paramAsInt("early_read_node_count", verifyRepositoryRequest.getEarlyReadNodeCount()) + ); verifyRepositoryRequest.seed(request.paramAsLong("seed", verifyRepositoryRequest.getSeed())); verifyRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", verifyRepositoryRequest.getMaxBlobSize())); verifyRepositoryRequest.timeout(request.paramAsTime("timeout", verifyRepositoryRequest.getTimeout())); From 0b78fe4863a0688ff9e51239247b7b00fa77b812 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sat, 16 Jan 2021 10:44:52 +0000 Subject: [PATCH 15/43] Remove another magic number --- .../org/elasticsearch/rest/RestRequest.java | 12 ++++++++ .../api/snapshot.repository_speed_test.json | 4 +++ .../rest-api-spec/test/10_speed_test.yml | 3 ++ .../testkit/RepositorySpeedTestAction.java | 29 +++++++++++++++++-- .../RestRepositorySpeedTestAction.java | 3 ++ 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/rest/RestRequest.java b/server/src/main/java/org/elasticsearch/rest/RestRequest.java index aedf3f026367a..a10a5834e4f75 100644 --- a/server/src/main/java/org/elasticsearch/rest/RestRequest.java +++ b/server/src/main/java/org/elasticsearch/rest/RestRequest.java @@ -367,6 +367,18 @@ public float paramAsFloat(String key, float defaultValue) { } } + public double paramAsDouble(String key, double defaultValue) { + String sValue = param(key); + if (sValue == null) { + return defaultValue; + } + try { + return Double.parseDouble(sValue); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Failed to parse double parameter [" + key + "] with value [" + sValue + "]", e); + } + } + public int paramAsInt(String key, int defaultValue) { String sValue = param(key); if (sValue == null) { diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json index 040bee4eaeaf7..def841478caf5 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json @@ -43,6 +43,10 @@ "type":"number", "description":"Seed for the random number generator used to create the test workload" }, + "rare_action_probability":{ + "type":"number", + "description":"Probability of taking a rare action such as an early read or an overwrite" + }, "max_blob_size":{ "type":"string", "description":"Maximum size of a blob to create during the test, e.g '1gb' or '100mb'" diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml index 8b84b38ea9b7f..a5b41638c9c94 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml @@ -48,6 +48,7 @@ setup: max_blob_size: 1mb read_node_count: 2 early_read_node_count: 1 + rare_action_probability: 0.01 - is_true: coordinating_node.id - is_true: coordinating_node.name @@ -56,6 +57,7 @@ setup: - match: { concurrency: 5 } - match: { read_node_count: 2 } - match: { early_read_node_count: 1 } + - match: { rare_action_probability: 0.01 } - match: { max_blob_size: 1mb } - is_true: seed - is_true: blob_path @@ -93,6 +95,7 @@ setup: - match: { concurrency: 5 } - match: { read_node_count: 10 } - match: { early_read_node_count: 2 } + - match: { rare_action_probability: 0.02 } - match: { max_blob_size: 1mb } - is_true: seed - is_true: blob_path diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java index b6b99d13acef9..adbb587cf9011 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java @@ -327,8 +327,8 @@ public void run() { nodes, request.readNodeCount, request.earlyReadNodeCount, - smallBlob && (random.nextInt(50) == 0), // TODO magic 50 - repository.supportURLRepo() && smallBlob && random.nextInt(50) == 0 // TODO magic 50 + smallBlob && random.nextDouble() < request.getRareActionProbability(), + repository.supportURLRepo() && smallBlob && random.nextDouble() < request.getRareActionProbability() ) ); queue.add(verifyBlobTask); @@ -507,6 +507,7 @@ private void sendResponse() { request.earlyReadNodeCount, request.maxBlobSize, request.seed, + request.rareActionProbability, blobPath, summary.build(), responses, @@ -551,6 +552,7 @@ public static class Request extends ActionRequest { private int readNodeCount = 10; private int earlyReadNodeCount = 2; private long seed = 0L; + private double rareActionProbability = 0.02; private TimeValue timeout = TimeValue.timeValueSeconds(30); private ByteSizeValue maxBlobSize = ByteSizeValue.ofMb(10); private boolean detailed = false; @@ -564,6 +566,7 @@ public Request(StreamInput in) throws IOException { super(in); repositoryName = in.readString(); seed = in.readLong(); + rareActionProbability = in.readDouble(); blobCount = in.readVInt(); concurrency = in.readVInt(); readNodeCount = in.readVInt(); @@ -584,6 +587,7 @@ public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeString(repositoryName); out.writeLong(seed); + out.writeDouble(rareActionProbability); out.writeVInt(blobCount); out.writeVInt(concurrency); out.writeVInt(readNodeCount); @@ -690,6 +694,19 @@ public int getEarlyReadNodeCount() { return earlyReadNodeCount; } + public void rareActionProbability(double rareActionProbability) { + if (rareActionProbability < 0. || rareActionProbability > 1.) { + throw new IllegalArgumentException( + "rareActionProbability must be between 0 and 1, but was [" + rareActionProbability + "]" + ); + } + this.rareActionProbability = rareActionProbability; + } + + public double getRareActionProbability() { + return rareActionProbability; + } + @Override public String toString() { return "Request{" + getDescription() + '}'; @@ -709,6 +726,8 @@ public String getDescription() { + earlyReadNodeCount + ", seed=" + seed + + ", rareActionProbability=" + + rareActionProbability + ", timeout=" + timeout + ", maxBlobSize=" @@ -737,6 +756,7 @@ public static class Response extends ActionResponse implements StatusToXContentO private final int earlyReadNodeCount; private final ByteSizeValue maxBlobSize; private final long seed; + private final double rareActionProbability; private final String blobPath; private final SpeedTestSummary summary; private final List blobResponses; @@ -753,6 +773,7 @@ public Response( int earlyReadNodeCount, ByteSizeValue maxBlobSize, long seed, + double rareActionProbability, String blobPath, SpeedTestSummary summary, List blobResponses, @@ -768,6 +789,7 @@ public Response( this.earlyReadNodeCount = earlyReadNodeCount; this.maxBlobSize = maxBlobSize; this.seed = seed; + this.rareActionProbability = rareActionProbability; this.blobPath = blobPath; this.summary = summary; this.blobResponses = blobResponses; @@ -786,6 +808,7 @@ public Response(StreamInput in) throws IOException { earlyReadNodeCount = in.readVInt(); maxBlobSize = new ByteSizeValue(in); seed = in.readLong(); + rareActionProbability = in.readDouble(); blobPath = in.readString(); summary = new SpeedTestSummary(in); blobResponses = in.readList(BlobSpeedTestAction.Response::new); @@ -804,6 +827,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(earlyReadNodeCount); maxBlobSize.writeTo(out); out.writeLong(seed); + out.writeDouble(rareActionProbability); out.writeString(blobPath); summary.writeTo(out); out.writeList(blobResponses); @@ -832,6 +856,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("early_read_node_count", earlyReadNodeCount); builder.field("max_blob_size", maxBlobSize); builder.field("seed", seed); + builder.field("rare_action_probability", rareActionProbability); builder.field("blob_path", blobPath); builder.field("summary", summary); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java index 7c1e5f7f42804..4630b446bbf67 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java @@ -39,6 +39,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC request.paramAsInt("early_read_node_count", verifyRepositoryRequest.getEarlyReadNodeCount()) ); verifyRepositoryRequest.seed(request.paramAsLong("seed", verifyRepositoryRequest.getSeed())); + verifyRepositoryRequest.rareActionProbability( + request.paramAsDouble("rare_action_probability", verifyRepositoryRequest.getRareActionProbability()) + ); verifyRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", verifyRepositoryRequest.getMaxBlobSize())); verifyRepositoryRequest.timeout(request.paramAsTime("timeout", verifyRepositoryRequest.getTimeout())); verifyRepositoryRequest.detailed(request.paramAsBoolean("detailed", verifyRepositoryRequest.getDetailed())); From 40e7e746836a1b18c2b6c817b1c31145f9648947 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sat, 16 Jan 2021 10:49:09 +0000 Subject: [PATCH 16/43] Document defaults --- .../api/snapshot.repository_speed_test.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json index def841478caf5..51ee076639a45 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json @@ -25,39 +25,39 @@ "params":{ "blob_count":{ "type":"number", - "description":"Number of blobs to create during the test" + "description":"Number of blobs to create during the test. Defaults to 100." }, "concurrency":{ "type":"number", - "description":"Number of operations to run concurrently during the test" + "description":"Number of operations to run concurrently during the test. Defaults to 10." }, "read_node_count":{ "type":"number", - "description":"Number of nodes on which to read a blob after writing" + "description":"Number of nodes on which to read a blob after writing. Defaults to 10." }, "early_read_node_count":{ "type":"number", - "description":"Number of nodes on which to read a blob during writing" + "description":"Number of nodes on which to perform an early read on a blob, i.e. before writing has completed. Early reads are rare actions so the 'rare_action_probability' parameter is also relevant. Defaults to 2." }, "seed":{ "type":"number", - "description":"Seed for the random number generator used to create the test workload" + "description":"Seed for the random number generator used to create the test workload. Defaults to a random value." }, "rare_action_probability":{ "type":"number", - "description":"Probability of taking a rare action such as an early read or an overwrite" + "description":"Probability of taking a rare action such as an early read or an overwrite. Defaults to 0.02." }, "max_blob_size":{ "type":"string", - "description":"Maximum size of a blob to create during the test, e.g '1gb' or '100mb'" + "description":"Maximum size of a blob to create during the test, e.g '1gb' or '100mb'. Defaults to '10mb'." }, "timeout":{ "type":"time", - "description":"Explicit operation timeout" + "description":"Explicit operation timeout. Defaults to '30s'." }, "detailed":{ "type":"boolean", - "description":"Whether to return detailed results or a summary. Defaults to `false`." + "description":"Whether to return detailed results or a summary. Defaults to 'false' so that only the summary is returned." } } } From 8db0096c6acf15a3a8038e9cf323271d330312d8 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sat, 16 Jan 2021 12:16:41 +0000 Subject: [PATCH 17/43] Record max read waiting time --- .../rest-api-spec/test/10_speed_test.yml | 22 ++++++++++--------- .../blobstore/testkit/SpeedTestSummary.java | 20 ++++++++++++----- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml index a5b41638c9c94..f43c982d8b16f 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml @@ -66,13 +66,14 @@ setup: - gte: { delete_nanos: 0} - match: { summary.write.count: 10} - gte: { summary.write.total_bytes: 0} - - gte: { summary.write.throttled_nanos: 0} - - gte: { summary.write.elapsed_nanos: 0} + - gte: { summary.write.total_throttled_nanos: 0} + - gte: { summary.write.total_elapsed_nanos: 0} - gte: { summary.read.count: 10} - gte: { summary.read.total_bytes: 0} - - gte: { summary.read.wait_nanos: 0} - - gte: { summary.read.throttled_nanos: 0} - - gte: { summary.read.elapsed_nanos: 0} + - gte: { summary.read.total_wait_nanos: 0} + - gte: { summary.read.max_wait_nanos: 0} + - gte: { summary.read.total_throttled_nanos: 0} + - gte: { summary.read.total_elapsed_nanos: 0} --- "Speed test with details": @@ -104,10 +105,11 @@ setup: - gte: { delete_nanos: 0} - match: { summary.write.count: 10} - gte: { summary.write.total_bytes: 0} - - gte: { summary.write.throttled_nanos: 0} - - gte: { summary.write.elapsed_nanos: 0} + - gte: { summary.write.total_throttled_nanos: 0} + - gte: { summary.write.total_elapsed_nanos: 0} - gte: { summary.read.count: 10} - gte: { summary.read.total_bytes: 0} - - gte: { summary.read.wait_nanos: 0} - - gte: { summary.read.throttled_nanos: 0} - - gte: { summary.read.elapsed_nanos: 0} + - gte: { summary.read.total_wait_nanos: 0} + - gte: { summary.read.max_wait_nanos: 0} + - gte: { summary.read.total_throttled_nanos: 0} + - gte: { summary.read.total_elapsed_nanos: 0} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java index 09d50b6391543..a8dfd16bcc189 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; +import java.util.concurrent.atomic.LongAccumulator; import java.util.concurrent.atomic.LongAdder; public class SpeedTestSummary implements Writeable, ToXContentFragment { @@ -24,6 +25,7 @@ public class SpeedTestSummary implements Writeable, ToXContentFragment { private final long readCount; private final long readBytes; private final long readWaitNanos; + private final long maxReadWaitNanos; private final long readThrottledNanos; private final long readElapsedNanos; @@ -35,6 +37,7 @@ public SpeedTestSummary( long readCount, long readBytes, long readWaitNanos, + long maxReadWaitNanos, long readThrottledNanos, long readElapsedNanos ) { @@ -45,6 +48,7 @@ public SpeedTestSummary( this.readCount = readCount; this.readBytes = readBytes; this.readWaitNanos = readWaitNanos; + this.maxReadWaitNanos = maxReadWaitNanos; this.readThrottledNanos = readThrottledNanos; this.readElapsedNanos = readElapsedNanos; } @@ -57,6 +61,7 @@ public SpeedTestSummary(StreamInput in) throws IOException { readCount = in.readVLong(); readBytes = in.readVLong(); readWaitNanos = in.readVLong(); + maxReadWaitNanos = in.readVLong(); readThrottledNanos = in.readVLong(); readElapsedNanos = in.readVLong(); } @@ -70,6 +75,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(readCount); out.writeVLong(readBytes); out.writeVLong(readWaitNanos); + out.writeVLong(maxReadWaitNanos); out.writeVLong(readThrottledNanos); out.writeVLong(readElapsedNanos); } @@ -81,16 +87,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject("write"); builder.field("count", writeCount); builder.field("total_bytes", writeBytes); - builder.field("throttled_nanos", writeThrottledNanos); - builder.field("elapsed_nanos", writeElapsedNanos); + builder.field("total_throttled_nanos", writeThrottledNanos); + builder.field("total_elapsed_nanos", writeElapsedNanos); builder.endObject(); builder.startObject("read"); builder.field("count", readCount); builder.field("total_bytes", readBytes); - builder.field("wait_nanos", readWaitNanos); - builder.field("throttled_nanos", readThrottledNanos); - builder.field("elapsed_nanos", readElapsedNanos); + builder.field("total_wait_nanos", readWaitNanos); + builder.field("max_wait_nanos", maxReadWaitNanos); + builder.field("total_throttled_nanos", readThrottledNanos); + builder.field("total_elapsed_nanos", readElapsedNanos); builder.endObject(); builder.endObject(); @@ -107,6 +114,7 @@ static class Builder { private final LongAdder readCount = new LongAdder(); private final LongAdder readBytes = new LongAdder(); private final LongAdder readWaitNanos = new LongAdder(); + private final LongAccumulator maxReadWaitNanos = new LongAccumulator(Long::max, Long.MIN_VALUE); private final LongAdder readThrottledNanos = new LongAdder(); private final LongAdder readElapsedNanos = new LongAdder(); @@ -119,6 +127,7 @@ public SpeedTestSummary build() { readCount.longValue(), readBytes.longValue(), readWaitNanos.longValue(), + Long.max(0L, maxReadWaitNanos.longValue()), readThrottledNanos.longValue(), readElapsedNanos.longValue() ); @@ -136,6 +145,7 @@ public void add(BlobSpeedTestAction.Response response) { readCount.add(1L); readBytes.add(checksumBytes); readWaitNanos.add(readDetail.getFirstByteNanos()); + maxReadWaitNanos.accumulate(readDetail.getFirstByteNanos()); readThrottledNanos.add(readDetail.getThrottledNanos()); readElapsedNanos.add(readDetail.getElapsedNanos()); } From 24d73920ee24a1ab27c8993b78095c51f4033450 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sat, 16 Jan 2021 13:22:28 +0000 Subject: [PATCH 18/43] Document response format --- .../apis/repo-speed-test-api.asciidoc | 264 +++++++++++++++++- .../testkit/BlobSpeedTestAction.java | 8 +- 2 files changed, 262 insertions(+), 10 deletions(-) diff --git a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc index ff1233d7d45b3..15922b3af2f35 100644 --- a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc @@ -118,17 +118,17 @@ after writing each blob. Defaults to `10`. operation while writing each blob. Defaults to `2`. Early read operations are only rarely performed. +`max_blob_size`:: +(Optional, <>) The maximum size of a blob to be written +during the test. Defaults to `10mb`. For realistic experiments you should set +this to at least `2gb`. + `seed`:: (Optional, integer) The seed for the pseudo-random number generator used to generate the list of operations performed during the test. To repeat the same set of operations in multiple experiments, use the same seed in each experiment. -`max_blob_size`:: -(Optional, <>) The maximum size of a blob to be written -during the test. Defaults to `10mb`. For realistic experiments you should set -this to at least `2gb`. - `timeout`:: (Optional, <>) Specifies the period of time to wait for the test to complete. If no response is received before the timeout expires, @@ -138,4 +138,256 @@ the test is cancelled and returns an error. Defaults to `30s`. [[repo-speed-test-api-response-body]] ==== {api-response-body-title} -TODO +`coordinating_node`:: +(object) +Identifies the node which coordinated the speed test and performed the final cleanup. ++ +.Properties of `coordinating_node` +[%collapsible%open] +==== +`id`:: +(string) +The id of the coordinating node. + +`name`:: +(string) +The name of the coordinating node +==== + +`repository`:: +(string) +The name of the repository that was the subject of the speed test. + +`blob_count`:: +(integer) +The number of blobs written to the repository during the test, equal to the +`?blob_count` request parameter. + +`concurrency`:: +(integer) +The number of write operations performed concurrently during the test, equal to +the `?concurrency` request parameter. + +`read_node_count`:: +(integer) +The limit on the number of nodes on which read operations were performed after +writing each blob, equal to the `?read_node_count` request parameter. + +`early_read_node_count`:: +(integer) +The limit on the number of nodes on which early read operations were performed +after writing each blob, equal to the `?early_read_node_count` request +parameter. + +`max_blob_size`:: +(string) +The limit on the size of a blob written during the test, equal to the +`?max_blob_size` parameter. + +`seed`:: +(long) +The seed for the pseudo-random number generator used to generate the operations +used during the test. Equal to the `?seed` request parameter if set. + +`rare_action_probability`:: +(double) +The probability of performing rare actions during the test. Equal to the +`?rare_action_probability` request parameter. + +`blob_path`:: +(string) +The path in the repository under which all the blobs were written during the +test. + +`summary`:: +(object) +A collection of statistics that summarise the results of the test. ++ +.Properties of `summary` +[%collapsible%open] +==== +`write`:: +(object) +A collection of statistics that summarise the results of the write operations +in the test. ++ +.Properties of `write` +[%collapsible%open] +===== +`count`:: +(integer) +The numer of write operations performed in the test. + +`total_bytes`:: +(integer) +The total size of all the blobs written in the test, in bytes. + +`total_throttled_nanos`:: +(integer) +The total time spent waiting due to the `max_snapshot_bytes_per_sec` throttle, +in nanoseconds. + +`total_elapsed_nanos`:: +(integer) +The total elapsed time spent on writing blobs in the test, in nanoseconds. +===== + +`read`:: +(object) +A collection of statistics that summarise the results of the read operations in +the test. ++ +.Properties of `read` +[%collapsible%open] +===== +`count`:: +(integer) +The numer of read operations performed in the test. + +`total_bytes`:: +(integer) +The total size of all the blobs or partial blobs read in the test, in bytes. + +`total_throttled_nanos`:: +(integer) +The total time spent waiting due to the `max_restore_bytes_per_sec` or +`indices.recovery.max_bytes_per_sec` throttles, in nanoseconds. + +`total_wait_nanos`:: +(integer) +The total time spent waiting for the first byte of each read request to be +received, in nanoseconds. + +`max_wait_nanos`:: +(integer) +The maximum time spent waiting for the first byte of any read request to be +received, in nanoseconds. + +`total_elapsed_nanos`:: +(integer) +The total elapsed time spent on reading blobs in the test, in nanoseconds. +===== +==== + +`details`:: +(array) +A description of every read and write operation performed during the test. This is +only returned if the `?details` request parameter is set to `true`. ++ +.Properties of items within `details` +[%collapsible] +==== +`blob`:: +(object) +A description of the blob that was written and read. ++ +.Properties of `blob` +[%collapsible%open] +===== +`name`:: +(string) +The name of the blob. + +`size`:: +(long) +The size of the blob in bytes. + +`read_start`:: +(long) +The position, in bytes, at which read operations started. + +`read_end`:: +(long) +The position, in bytes, at which read operations completed. + +`read_early`:: +(boolean) +Whether any read operations were started before the write operation completed. + +`overwritten`:: +(boolean) +Whether the blob was overwritten while the read operations were ongoing. +===== + +`writer_node`:: +(object) +Identifies the node which wrote this blob and coordinated the read operations. ++ +.Properties of `writer_node` +[%collapsible%open] +===== +`id`:: +(string) +The id of the writer node. + +`name`:: +(string) +The name of the writer node +===== + +`write_elapsed_nanos`:: +(object) +The elapsed time spent writing this blob, in nanoseconds. + +`overwrite_elapsed_nanos`:: +(object) +The elapsed time spent overwriting this blob, in nanoseconds. Omitted if the +blob was not overwritten. + +`write_throttled_nanos`:: +(object) +The length of time spent waiting for the `max_snapshot_bytes_per_sec` throttle +while writing this blob, in nanoseconds. + +`reads`:: +(array) +A description of every read operation performed on this blob. ++ +.Properties of items within `reads` +[%collapsible%open] +===== +`node`:: +(object) +Identifies the node which performed the read operation. ++ +.Properties of `node` +[%collapsible%open] +====== +`id`:: +(string) +The id of the reader node. + +`name`:: +(string) +The name of the reader node +====== + +`before_write_complete`:: +(boolean) +Whether the read operation may have started before the write operation was +complete. Omitted if `false`. + +`found`:: +(boolean) +Whether the blob was found by this read operation or not. May be `false` if the +read was started before the write completed. + +`first_byte_nanos`:: +(boolean) +The length of time waiting for the first byte of the read operation to be +received, in nanoseconds. Omitted if the blob was not found. + +`elapsed_nanos`:: +(boolean) +The length of time spent reading this blob, in nanoseconds. Omitted if the blob +was not found. + +`throttled_nanos`:: +(boolean) +The length of time time spent waiting due to the `max_restore_bytes_per_sec` or +`indices.recovery.max_bytes_per_sec` throttles during the read of this blob, in +nanoseconds. Omitted if the blob was not found. + +===== + +==== diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java index 02acb25615cf1..56ec10c1ab95c 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java @@ -800,9 +800,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("name", nodeName); builder.endObject(); - builder.field("write_nanos", writeElapsedNanos); + builder.field("write_elapsed_nanos", writeElapsedNanos); if (overwrite) { - builder.field("overwrite_nanos", overwriteElapsedNanos); + builder.field("overwrite_elapsed_nanos", overwriteElapsedNanos); } builder.field("write_throttled_nanos", writeThrottledNanos); @@ -904,8 +904,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } else { builder.field("found", true); builder.field("first_byte_nanos", firstByteNanos); - builder.field("read_nanos", elapsedNanos); - builder.field("throttle_nanos", throttleNanos); + builder.field("elapsed_nanos", elapsedNanos); + builder.field("throttled_nanos", throttleNanos); } builder.endObject(); From cbe503b64a623d34e4820121d0cbee919e18a207 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 08:11:01 +0000 Subject: [PATCH 19/43] License headers --- .../blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java | 5 +++-- .../rest/SnapshotRepoTestKitClientYamlTestSuiteIT.java | 5 +++-- .../blobstore/testkit/RepositorySpeedTestIT.java | 5 +++-- .../repositories/blobstore/testkit/BlobSpeedTestAction.java | 5 +++-- .../blobstore/testkit/GetBlobChecksumAction.java | 5 +++-- .../repositories/blobstore/testkit/RandomBlobContent.java | 5 +++-- .../blobstore/testkit/RandomBlobContentBytesReference.java | 5 +++-- .../blobstore/testkit/RandomBlobContentStream.java | 5 +++-- .../blobstore/testkit/RepositorySpeedTestAction.java | 5 +++-- .../blobstore/testkit/RestRepositorySpeedTestAction.java | 5 +++-- .../blobstore/testkit/SnapshotRepositoryTestKit.java | 5 +++-- .../repositories/blobstore/testkit/SpeedTestSummary.java | 5 +++-- .../testkit/AbstractSnapshotRepoTestKitRestTestCase.java | 5 +++-- .../testkit/RandomBlobContentBytesReferenceTests.java | 5 +++-- .../blobstore/testkit/RandomBlobContentStreamTests.java | 5 +++-- 15 files changed, 45 insertions(+), 30 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java index e56a4e2de4d37..77dfb3902805a 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/FsSnapshotRepoTestKitIT.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit.rest; diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/SnapshotRepoTestKitClientYamlTestSuiteIT.java b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/SnapshotRepoTestKitClientYamlTestSuiteIT.java index bb20d36c73181..a5c1daeda11fa 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/SnapshotRepoTestKitClientYamlTestSuiteIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/java/org/elasticsearch/repositories/blobstore/testkit/rest/SnapshotRepoTestKitClientYamlTestSuiteIT.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit.rest; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java index 359c98f263a34..df709dd9180cc 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java index 56ec10c1ab95c..dc07d739fe15f 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java index dc7576e2460bc..53af4a4ab3ece 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java index f7a1c21faa503..730b514a64e78 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java index f856431b5134d..9046e1e0cb60d 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReference.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java index 496e089c0b22a..c6163a7ffd82d 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStream.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java index adbb587cf9011..f2c85761435bb 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestAction.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java index 4630b446bbf67..5375280190ba2 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java index 60e5607cb9643..0d60f4beea058 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java index a8dfd16bcc189..dca306512bb32 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java index fa1397262a094..7c49e0a3f0b29 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java index 70c88a357967f..d9a056ba60be0 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentBytesReferenceTests.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java index 5d37b2fe2278e..00ea83f8cbd9c 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContentStreamTests.java @@ -1,7 +1,8 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. */ package org.elasticsearch.repositories.blobstore.testkit; From fbb0ed87eb495d0afc2720c4fb0e82322c85f578 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 08:21:39 +0000 Subject: [PATCH 20/43] Rename speed test -> analysis --- ...pi.asciidoc => repo-analysis-api.asciidoc} | 59 ++- .../apis/snapshot-restore-apis.asciidoc | 2 +- .../register-repository.asciidoc | 2 +- .../CompatibleVersionHelperTests.java.new | 389 ------------------ .../operator/OperatorOnlyRegistry.java | 9 +- ....json => snapshot.repository_analyse.json} | 6 +- .../{10_speed_test.yml => 10_analyse.yml} | 12 +- ...dTestIT.java => RepositoryAnalysisIT.java} | 14 +- ...TestAction.java => BlobAnalyseAction.java} | 38 +- .../testkit/GetBlobChecksumAction.java | 2 +- ...tion.java => RepositoryAnalyseAction.java} | 77 ++-- ...java => RepositoryPerformanceSummary.java} | 14 +- .../testkit/RestRepositoryAnalyseAction.java | 57 +++ .../RestRepositorySpeedTestAction.java | 57 --- .../testkit/SnapshotRepositoryTestKit.java | 6 +- ...stractSnapshotRepoTestKitRestTestCase.java | 4 +- 16 files changed, 178 insertions(+), 570 deletions(-) rename docs/reference/snapshot-restore/apis/{repo-speed-test-api.asciidoc => repo-analysis-api.asciidoc} (83%) delete mode 100644 server/src/test/java/org/elasticsearch/rest/CompatibleVersionHelperTests.java.new rename x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/{snapshot.repository_speed_test.json => snapshot.repository_analyse.json} (92%) rename x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/{10_speed_test.yml => 10_analyse.yml} (93%) rename x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/{RepositorySpeedTestIT.java => RepositoryAnalysisIT.java} (94%) rename x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/{BlobSpeedTestAction.java => BlobAnalyseAction.java} (96%) rename x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/{RepositorySpeedTestAction.java => RepositoryAnalyseAction.java} (92%) rename x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/{SpeedTestSummary.java => RepositoryPerformanceSummary.java} (92%) create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java delete mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java diff --git a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc similarity index 83% rename from docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc rename to docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc index 15922b3af2f35..73eba9fb83ea6 100644 --- a/docs/reference/snapshot-restore/apis/repo-speed-test-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc @@ -1,16 +1,12 @@ [role="xpack"] -[[repo-speed-test-api]] -=== Repository speed test API +[[repo-analysis-api]] +=== Repository analysis API ++++ -Repository speed test +Repository analysis ++++ -// TODO TBD I originally styled this as an "extended verification API" but feel -// that verification is too strong a word. We certainly report errors if it -// finds any, but no errors doesn't mean the repository is 100% correct, nor -// does it meet any performance threshold. A speed test implies weaker claims -// so I expect it would be less misleading. -Measures the performance characteristics of a snapshot repository. +Analyses a repository, reporting its performance characteristics and any +incorrect behaviour found. //// [source,console] @@ -28,28 +24,27 @@ PUT /_snapshot/my_repository [source,console] ---- -POST /_snapshot/my_repository/_speed_test?blob_count=10&concurrency=4&max_blob_size=1mb&timeout=120s +POST /_snapshot/my_repository/_analyse?blob_count=10&concurrency=4&max_blob_size=1mb&timeout=120s ---- -[[repo-speed-test-api-request]] +[[repo-analysis-api-request]] ==== {api-request-title} -`POST /_snapshot//_speed_test` +`POST /_snapshot//_analyse` -[[repo-speed-test-api-desc]] +[[repo-analysis-api-desc]] ==== {api-description-title} There are a large number of third-party storage systems available, not all of which are suitable for use as a snapshot repository by {es}. Some storage -systems perform poorly, or behave incorrectly, especially when accessed +systems behave incorrectly, or perform poorly, especially when accessed concurrently by multiple clients as the nodes of an {es} cluster do. -The Repository speed test API performs a collection of read and write -operations on your repository which are specially designed to detect incorrect -behaviour and to measure the performance characteristics of your storage -system. +The Repository analysis API performs a collection of read and write operations +on your repository which are specially designed to detect incorrect behaviour +and to measure the performance characteristics of your storage system. -Each speed test runs a wide variety of operations generated by a pseudo-random +Each analysis runs a wide variety of operations generated by a pseudo-random process. You can seed this process using the optional `seed` parameter in order to repeat the same set of operations in multiple experiments. Note that the operations are performed concurrently so may not always happen in the same @@ -61,9 +56,9 @@ should set `blob_count` to at least `2000` and `max_blob_size` to at least `2gb`, and will almost certainly need to increase the `timeout` to allow time for the process to complete successfully. -If the speed test is successful this API returns details of the testing -process, including how long each operation took. You can use this information -to analyse the performance of your storage system. If any operation fails or +If the analysis is successful this API returns details of the testing process, +including how long each operation took. You can use this information to +determine the performance of your storage system. If any operation fails or returns an incorrect result, this API returns an error. If the API returns an error then it may not have removed all the data it wrote to the repository. The error will indicate the location of any leftover data, and this path is also @@ -72,32 +67,32 @@ been cleaned up correctly. If there is still leftover data at the specified location then you should manually remove it. If the connection from your client to {es} is closed while the client is -waiting for the result of the speed test then the test is cancelled. Since a -speed test takes a long time to complete, you may need to configure your client -to wait for longer than usual for a response. On cancellation the speed test +waiting for the result of the analysis then the test is cancelled. Since ab +analysis takes a long time to complete, you may need to configure your client +to wait for longer than usual for a response. On cancellation the analysis attempts to clean up the data it was writing, but it may not be able to remove it all. The path to the leftover data is recorded in the {es} logs. You should verify yourself that this location has been cleaned up correctly. If there is still leftover data at the specified location then you should manually remove it. -NOTE: A speed test writes a substantial amount of data to your repository and +NOTE: An analysis writes a substantial amount of data to your repository and then reads it back again. This consumes bandwidth on the network between the cluster and the repository, and storage space and IO bandwidth on the repository itself. You must ensure this load does not affect other users of -these systems. Speed tests respect the repository settings +these systems. Analyses respect the repository settings `max_snapshot_bytes_per_sec` and `max_restore_bytes_per_sec` if available, and the cluster setting `indices.recovery.max_bytes_per_sec` which you can use to limit the bandwidth they consume. -[[repo-speed-test-api-path-params]] +[[repo-analysis-api-path-params]] ==== {api-path-parms-title} ``:: (Required, string) Name of the snapshot repository to test. -[[repo-speed-test-api-query-params]] +[[repo-analysis-api-query-params]] ==== {api-query-parms-title} `blob_count`:: @@ -135,12 +130,12 @@ the test to complete. If no response is received before the timeout expires, the test is cancelled and returns an error. Defaults to `30s`. [role="child_attributes"] -[[repo-speed-test-api-response-body]] +[[repo-analysis-api-response-body]] ==== {api-response-body-title} `coordinating_node`:: (object) -Identifies the node which coordinated the speed test and performed the final cleanup. +Identifies the node which coordinated the analysis and performed the final cleanup. + .Properties of `coordinating_node` [%collapsible%open] @@ -156,7 +151,7 @@ The name of the coordinating node `repository`:: (string) -The name of the repository that was the subject of the speed test. +The name of the repository that was the subject of the analysis. `blob_count`:: (integer) diff --git a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc index 8ffd1834b3b73..5627d5a734d76 100644 --- a/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc +++ b/docs/reference/snapshot-restore/apis/snapshot-restore-apis.asciidoc @@ -27,7 +27,7 @@ For more information, see <>. include::put-repo-api.asciidoc[] include::verify-repo-api.asciidoc[] -include::repo-speed-test-api.asciidoc[] +include::repo-analysis-api.asciidoc[] include::get-repo-api.asciidoc[] include::delete-repo-api.asciidoc[] include::clean-up-repo-api.asciidoc[] diff --git a/docs/reference/snapshot-restore/register-repository.asciidoc b/docs/reference/snapshot-restore/register-repository.asciidoc index b480b64b3a7ee..686e524f37c31 100644 --- a/docs/reference/snapshot-restore/register-repository.asciidoc +++ b/docs/reference/snapshot-restore/register-repository.asciidoc @@ -261,7 +261,7 @@ POST /_snapshot/my_unverified_backup/_verify It returns a list of nodes where repository was successfully verified or an error message if verification process failed. If desired, you can also test a repository more thoroughly using the -<>. +<>. [discrete] [[snapshots-repository-cleanup]] diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleVersionHelperTests.java.new b/server/src/test/java/org/elasticsearch/rest/CompatibleVersionHelperTests.java.new deleted file mode 100644 index f96451e74d37f..0000000000000 --- a/server/src/test/java/org/elasticsearch/rest/CompatibleVersionHelperTests.java.new +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.rest; - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.Version; -import org.elasticsearch.common.xcontent.ParsedMediaType; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.hamcrest.ElasticsearchMatchers; -import org.hamcrest.Matcher; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.nullValue; - -public class CompatibleVersionHelperTests extends ESTestCase { - int CURRENT_VERSION = Version.CURRENT.major; - int PREVIOUS_VERSION = Version.CURRENT.major - 1; - int OBSOLETE_VERSION = Version.CURRENT.major - 2; - - public void testAcceptAndContentTypeCombinations() { - assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()), isCompatible()); - - assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()), isCompatible()); - - ElasticsearchStatusException e = expectThrows( - ElasticsearchStatusException.class, - () -> requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent()) - ); - assertThat( - e.getMessage(), - equalTo( - "A compatible version is required on both Content-Type and Accept headers " - + "if either one has requested a compatible version and the compatible versions must match. " - + "Accept=" - + acceptHeader(PREVIOUS_VERSION) - + ", Content-Type=" - + contentTypeHeader(CURRENT_VERSION) - ) - ); - - // no body - content-type is ignored - assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent()), isCompatible()); - // no body - content-type is ignored - assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()), not(isCompatible())); - - e = expectThrows( - ElasticsearchStatusException.class, - () -> requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()) - ); - assertThat( - e.getMessage(), - equalTo( - "A compatible version is required on both Content-Type and Accept headers " - + "if either one has requested a compatible version and the compatible versions must match. " - + "Accept=" - + acceptHeader(CURRENT_VERSION) - + ", Content-Type=" - + contentTypeHeader(PREVIOUS_VERSION) - ) - ); - - assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent()), not(isCompatible())); - - assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent()), not(isCompatible())); - - // tests when body present and one of the headers missing - versioning is required on both when body is present - e = expectThrows( - ElasticsearchStatusException.class, - () -> requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent()) - ); - assertThat( - e.getMessage(), - equalTo( - "A compatible version is required on both Content-Type and Accept headers " - + "if either one has requested a compatible version and the compatible versions must match. " - + "Accept=" - + acceptHeader(PREVIOUS_VERSION) - + ", Content-Type=" - + contentTypeHeader(null) - ) - ); - - e = expectThrows( - ElasticsearchStatusException.class, - () -> requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent()) - ); - assertThat( - e.getMessage(), - equalTo( - "A compatible version is required on both Content-Type and Accept headers " - + "if either one has requested a compatible version. " - + "Accept=" - + acceptHeader(CURRENT_VERSION) - + ", Content-Type=" - + contentTypeHeader(null) - ) - ); - - e = expectThrows( - ElasticsearchStatusException.class, - () -> requestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent()) - ); - assertThat( - e.getMessage(), - equalTo( - "A compatible version is required on both Content-Type and Accept headers " - + "if either one has requested a compatible version. " - + "Accept=" - + acceptHeader(null) - + ", Content-Type=" - + contentTypeHeader(CURRENT_VERSION) - ) - ); - - e = expectThrows( - ElasticsearchStatusException.class, - () -> requestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()) - ); - assertThat( - e.getMessage(), - equalTo( - "A compatible version is required on both Content-Type and Accept headers " - + "if either one has requested a compatible version and the compatible versions must match. " - + "Accept=" - + acceptHeader(null) - + ", Content-Type=" - + contentTypeHeader(PREVIOUS_VERSION) - ) - ); - - // tests when body NOT present and one of the headers missing - assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyNotPresent()), isCompatible()); - - assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyNotPresent()), not(isCompatible())); - - // body not present - accept header is missing - it will default to Current version. Version on content type is ignored - assertThat(requestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()), not(isCompatible())); - - assertThat(requestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyNotPresent()), not(isCompatible())); - - assertThat(requestWith(acceptHeader(null), contentTypeHeader(null), bodyNotPresent()), not(isCompatible())); - - // Accept header = application/json means current version. If body is provided then accept and content-Type should be the same - assertThat(requestWith(acceptHeader("application/json"), contentTypeHeader(null), bodyNotPresent()), not(isCompatible())); - - assertThat( - requestWith(acceptHeader("application/json"), contentTypeHeader("application/json"), bodyPresent()), - not(isCompatible()) - ); - - assertThat(requestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyPresent()), not(isCompatible())); - } - - public void testObsoleteVersion() { - ElasticsearchStatusException e = expectThrows( - ElasticsearchStatusException.class, - () -> requestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(OBSOLETE_VERSION), bodyPresent()) - ); - assertThat( - e.getMessage(), - equalTo( - "Accept version must be either version " - + CURRENT_VERSION - + " or " - + PREVIOUS_VERSION - + ", but found " - + OBSOLETE_VERSION - + ". " - + "Accept=" - + acceptHeader(OBSOLETE_VERSION) - ) - ); - - e = expectThrows( - ElasticsearchStatusException.class, - () -> requestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(null), bodyNotPresent()) - ); - assertThat( - e.getMessage(), - equalTo( - "Accept version must be either version " - + CURRENT_VERSION - + " or " - + PREVIOUS_VERSION - + ", but found " - + OBSOLETE_VERSION - + ". " - + "Accept=" - + acceptHeader(OBSOLETE_VERSION) - ) - ); - - e = expectThrows( - ElasticsearchStatusException.class, - () -> requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(OBSOLETE_VERSION), bodyPresent()) - ); - assertThat( - e.getMessage(), - equalTo( - "Content-Type version must be either version " - + CURRENT_VERSION - + " or " - + PREVIOUS_VERSION - + ", but found " - + OBSOLETE_VERSION - + ". " - + "Content-Type=" - + contentTypeHeader(OBSOLETE_VERSION) - ) - ); - } - - public void testMediaTypeCombinations() { - // body not present - ignore content-type - assertThat(requestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()), not(isCompatible())); - - assertThat(requestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyNotPresent()), not(isCompatible())); - - assertThat(requestWith(acceptHeader("*/*"), contentTypeHeader("application/json"), bodyNotPresent()), not(isCompatible())); - - // this is for instance used by SQL - assertThat( - requestWith(acceptHeader("application/json"), contentTypeHeader("application/cbor"), bodyPresent()), - not(isCompatible()) - ); - - assertThat( - requestWith( - acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"), - contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=7"), - bodyPresent() - ), - isCompatible() - ); - - // different versions on different media types - expectThrows( - ElasticsearchStatusException.class, - () -> requestWith( - acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"), - contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=8"), - bodyPresent() - ) - ); - } - - public void testTextMediaTypes() { - assertThat( - requestWith(acceptHeader("text/tab-separated-values"), contentTypeHeader("application/json"), bodyNotPresent()), - not(isCompatible()) - ); - - assertThat(requestWith(acceptHeader("text/plain"), contentTypeHeader("application/json"), bodyNotPresent()), not(isCompatible())); - - assertThat(requestWith(acceptHeader("text/csv"), contentTypeHeader("application/json"), bodyNotPresent()), not(isCompatible())); - - // versioned - assertThat( - requestWith( - acceptHeader("text/vnd.elasticsearch+tab-separated-values;compatible-with=7"), - contentTypeHeader(7), - bodyNotPresent() - ), - isCompatible() - ); - - assertThat( - requestWith(acceptHeader("text/vnd.elasticsearch+plain;compatible-with=7"), contentTypeHeader(7), bodyNotPresent()), - isCompatible() - ); - - assertThat( - requestWith(acceptHeader("text/vnd.elasticsearch+csv;compatible-with=7"), contentTypeHeader(7), bodyNotPresent()), - isCompatible() - ); - } - - public void testVersionParsing() { - byte version = randomNonNegativeByte(); - assertThat( - CompatibleVersionHelper.parseVersion( - ParsedMediaType.parseMediaType("application/vnd.elasticsearch+json;compatible-with=" + version) - ), - equalTo(version) - ); - assertThat( - CompatibleVersionHelper.parseVersion( - ParsedMediaType.parseMediaType("application/vnd.elasticsearch+cbor;compatible-with=" + version) - ), - equalTo(version) - ); - assertThat( - CompatibleVersionHelper.parseVersion( - ParsedMediaType.parseMediaType("application/vnd.elasticsearch+smile;compatible-with=" + version) - ), - equalTo(version) - ); - assertThat( - CompatibleVersionHelper.parseVersion( - ParsedMediaType.parseMediaType("application/vnd.elasticsearch+x-ndjson;compatible-with=" + version) - ), - equalTo(version) - ); - assertThat(CompatibleVersionHelper.parseVersion(ParsedMediaType.parseMediaType("application/json")), nullValue()); - - assertThat( - CompatibleVersionHelper.parseVersion( - ParsedMediaType.parseMediaType("APPLICATION/VND.ELASTICSEARCH+JSON;COMPATIBLE-WITH=" + version) - ), - equalTo(version) - ); - assertThat(CompatibleVersionHelper.parseVersion(ParsedMediaType.parseMediaType("APPLICATION/JSON")), nullValue()); - - assertThat(CompatibleVersionHelper.parseVersion(ParsedMediaType.parseMediaType("application/json; sth=123")), is(nullValue())); - - } - - private Matcher isCompatible() { - return requestHasVersion(PREVIOUS_VERSION); - } - - private Matcher requestHasVersion(int version) { - return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(v -> (int) v.major, equalTo(version)); - } - - private String bodyNotPresent() { - return ""; - } - - private String bodyPresent() { - return "some body"; - } - - private String contentTypeHeader(int version) { - return mediaType(String.valueOf(version)); - } - - private String acceptHeader(int version) { - return mediaType(String.valueOf(version)); - } - - private String acceptHeader(String value) { - return value; - } - - private String contentTypeHeader(String value) { - return value; - } - - private String mediaType(String version) { - if (version != null) { - return "application/vnd.elasticsearch+json;compatible-with=" + version; - } - return null; - } - - private Version requestWith(String accept, String contentType, String body) { - ParsedMediaType parsedAccept = ParsedMediaType.parseMediaType(accept); - ParsedMediaType parsedContentType = ParsedMediaType.parseMediaType(contentType); - return CompatibleVersionHelper.getCompatibleVersion(parsedAccept, parsedContentType, body.isEmpty() == false); - } - -} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java index 905d05f290a32..744755f1e699f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java @@ -34,11 +34,10 @@ public class OperatorOnlyRegistry { "cluster:admin/autoscaling/delete_autoscaling_policy", "cluster:admin/autoscaling/get_autoscaling_policy", "cluster:admin/autoscaling/get_autoscaling_capacity", - // Repository speed test actions are not mentioned in core, literal strings are needed. - // TODO consider whether these are operator-only actions - "cluster:admin/repository/speed_test", - "cluster:admin/repository/speed_test/blob", - "cluster:admin/repository/speed_test/blob/read" + // Repository analysis actions are not mentioned in core, literal strings are needed. + "cluster:admin/repository/analyse", + "cluster:admin/repository/analyse/blob", + "cluster:admin/repository/analyse/blob/read" ); private final ClusterSettings clusterSettings; diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json similarity index 92% rename from x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json rename to x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json index 51ee076639a45..5439d38995ee3 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_speed_test.json +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json @@ -1,15 +1,15 @@ { - "snapshot.repository_speed_test":{ + "snapshot.repository_analyse":{ "documentation":{ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html", - "description":"Performs a speed test on a repository" + "description":"Analyses a repository for correctness and performance" }, "stability":"stable", "visibility":"public", "url":{ "paths":[ { - "path":"/_snapshot/{repository}/_speed_test", + "path":"/_snapshot/{repository}/_analyse", "methods":[ "POST" ], diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml similarity index 93% rename from x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml rename to x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml index f43c982d8b16f..cc1d8b95700da 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_speed_test.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml @@ -19,14 +19,14 @@ setup: location: "test_repo_loc" --- -"Speed test fails on readonly repositories": +"Analysis fails on readonly repositories": - skip: version: "- 7.99.99" reason: "introduced in 8.0" - do: catch: bad_request - snapshot.repository_speed_test: + snapshot.repository_analyse: repository: test_repo_readonly - match: { status: 400 } @@ -35,13 +35,13 @@ setup: --- -"Speed test without details": +"Analysis without details": - skip: version: "- 7.99.99" reason: "introduced in 8.0" - do: - snapshot.repository_speed_test: + snapshot.repository_analyse: repository: test_repo blob_count: 10 concurrency: 5 @@ -76,13 +76,13 @@ setup: - gte: { summary.read.total_elapsed_nanos: 0} --- -"Speed test with details": +"Analysis with details": - skip: version: "- 7.99.99" reason: "introduced in 8.0" - do: - snapshot.repository_speed_test: + snapshot.repository_analyse: repository: test_repo blob_count: 10 concurrency: 5 diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java similarity index 94% rename from x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java rename to x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java index df709dd9180cc..ad990f6979aea 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositorySpeedTestIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java @@ -50,7 +50,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.startsWith; -public class RepositorySpeedTestIT extends AbstractSnapshotIntegTestCase { +public class RepositoryAnalysisIT extends AbstractSnapshotIntegTestCase { // These tests are incomplete and do not currently verify that the speed test picks up on any particular repository-side failures // TODO complete these tests @@ -78,7 +78,7 @@ public void testFoo() { } } - final RepositorySpeedTestAction.Request request = new RepositorySpeedTestAction.Request("test-repo"); + final RepositoryAnalyseAction.Request request = new RepositoryAnalyseAction.Request("test-repo"); if (randomBoolean()) { request.concurrency(between(1, 5)); @@ -98,7 +98,7 @@ public void testFoo() { request.timeout(TimeValue.timeValueSeconds(5)); - final RepositorySpeedTestAction.Response response = client().execute(RepositorySpeedTestAction.INSTANCE, request).actionGet(); + final RepositoryAnalyseAction.Response response = client().execute(RepositoryAnalyseAction.INSTANCE, request).actionGet(); assertThat(response.status(), equalTo(RestStatus.OK)); } @@ -164,13 +164,13 @@ public String startVerification() { static class DisruptableBlobStore implements BlobStore { private final Map blobContainers = ConcurrentCollections.newConcurrentMap(); - private Semaphore writeSemaphore = new Semaphore(new RepositorySpeedTestAction.Request("dummy").getConcurrency()); - private int maxBlobCount = new RepositorySpeedTestAction.Request("dummy").getBlobCount(); - private long maxBlobSize = new RepositorySpeedTestAction.Request("dummy").getMaxBlobSize().getBytes(); + private Semaphore writeSemaphore = new Semaphore(new RepositoryAnalyseAction.Request("dummy").getConcurrency()); + private int maxBlobCount = new RepositoryAnalyseAction.Request("dummy").getBlobCount(); + private long maxBlobSize = new RepositoryAnalyseAction.Request("dummy").getMaxBlobSize().getBytes(); @Override public BlobContainer blobContainer(BlobPath path) { - assertThat(path.buildAsString(), startsWith("temp-speed-test-")); + assertThat(path.buildAsString(), startsWith("temp-analysis-")); return blobContainers.computeIfAbsent( path.buildAsString(), ignored -> new DisruptableBlobContainer(path, this::deleteContainer, writeSemaphore, maxBlobCount, maxBlobSize) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java similarity index 96% rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java index dc07d739fe15f..f6960e38b41b3 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobSpeedTestAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java @@ -140,20 +140,20 @@ * unnecessary resources. * */ -public class BlobSpeedTestAction extends ActionType { +public class BlobAnalyseAction extends ActionType { - private static final Logger logger = LogManager.getLogger(BlobSpeedTestAction.class); + private static final Logger logger = LogManager.getLogger(BlobAnalyseAction.class); - public static final BlobSpeedTestAction INSTANCE = new BlobSpeedTestAction(); - public static final String NAME = "cluster:admin/repository/speed_test/blob"; + public static final BlobAnalyseAction INSTANCE = new BlobAnalyseAction(); + public static final String NAME = "cluster:admin/repository/analyse/blob"; - private BlobSpeedTestAction() { + private BlobAnalyseAction() { super(NAME, Response::new); } public static class TransportAction extends HandledTransportAction { - private static final Logger logger = BlobSpeedTestAction.logger; + private static final Logger logger = BlobAnalyseAction.logger; private final RepositoriesService repositoriesService; private final TransportService transportService; @@ -180,20 +180,20 @@ protected void doExecute(Task task, Request request, ActionListener li logger.trace("handling [{}]", request); assert task instanceof CancellableTask; - new BlobSpeedTest(transportService, (CancellableTask) task, request, blobStoreRepository, blobContainer, listener).run(); + new BlobAnalysis(transportService, (CancellableTask) task, request, blobStoreRepository, blobContainer, listener).run(); } } /** - * Speed test of a single blob, performing the write(s) and orchestrating the read(s). + * Analysis on a single blob, performing the write(s) and orchestrating the read(s). */ - static class BlobSpeedTest { + static class BlobAnalysis { private final TransportService transportService; private final CancellableTask task; - private final BlobSpeedTestAction.Request request; + private final BlobAnalyseAction.Request request; private final BlobStoreRepository repository; private final BlobContainer blobContainer; - private final ActionListener listener; + private final ActionListener listener; private final Random random; private final boolean checksumWholeBlob; private final long checksumStart; @@ -204,13 +204,13 @@ static class BlobSpeedTest { private final StepListener write1Step = new StepListener<>(); private final StepListener write2Step = new StepListener<>(); - BlobSpeedTest( + BlobAnalysis( TransportService transportService, CancellableTask task, - BlobSpeedTestAction.Request request, + BlobAnalyseAction.Request request, BlobStoreRepository repository, BlobContainer blobContainer, - ActionListener listener + ActionListener listener ) { this.transportService = transportService; this.task = task; @@ -414,7 +414,7 @@ private void cancelReadsCleanUpAndReturnFailure(Exception exception) { private void cleanUpAndReturnFailure(Exception exception) { if (logger.isTraceEnabled()) { - logger.trace(new ParameterizedMessage("speed test failed [{}] cleaning up", request.getDescription()), exception); + logger.trace(new ParameterizedMessage("analysis failed [{}] cleaning up", request.getDescription()), exception); } try { blobContainer.deleteBlobsIgnoringIfNotExists(List.of(request.blobName)); @@ -422,7 +422,7 @@ private void cleanUpAndReturnFailure(Exception exception) { exception.addSuppressed(ioException); logger.warn( new ParameterizedMessage( - "failure during post-failure cleanup while speed-testing repository [{}], you may need to manually remove [{}/{}]", + "failure during post-failure cleanup while analysing repository [{}], you may need to manually remove [{}/{}]", request.getRepositoryName(), request.getBlobPath(), request.getBlobName() @@ -482,7 +482,7 @@ private void onReadsComplete(Collection responses, WriteDetails wr request.getRepositoryName(), "node [" + nodeResponse.node - + "] failed during speed test: expected to read [" + + "] failed during analysis: expected to read [" + checksumStart + "-" + checksumEnd @@ -653,7 +653,7 @@ public ActionRequestValidationException validate() { @Override public String getDescription() { - return "blob speed test [" + return "blob analysis [" + repositoryName + ":" + blobPath @@ -672,7 +672,7 @@ public String getDescription() { @Override public String toString() { - return "BlobSpeedTestAction.Request{" + getDescription() + "}"; + return "BlobAnalyseAction.Request{" + getDescription() + "}"; } @Override diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java index 53af4a4ab3ece..d843e46ecfca5 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java @@ -52,7 +52,7 @@ public class GetBlobChecksumAction extends ActionType { +public class RepositoryAnalyseAction extends ActionType { - private static final Logger logger = LogManager.getLogger(RepositorySpeedTestAction.class); + private static final Logger logger = LogManager.getLogger(RepositoryAnalyseAction.class); - public static final RepositorySpeedTestAction INSTANCE = new RepositorySpeedTestAction(); - public static final String NAME = "cluster:admin/repository/speed_test"; + public static final RepositoryAnalyseAction INSTANCE = new RepositoryAnalyseAction(); + public static final String NAME = "cluster:admin/repository/analyse"; - private RepositorySpeedTestAction() { + private RepositoryAnalyseAction() { super(NAME, Response::new); } @@ -98,7 +98,7 @@ public TransportAction( ClusterService clusterService, RepositoriesService repositoriesService ) { - super(NAME, transportService, actionFilters, RepositorySpeedTestAction.Request::new, ThreadPool.Names.SAME); + super(NAME, transportService, actionFilters, RepositoryAnalyseAction.Request::new, ThreadPool.Names.SAME); this.transportService = transportService; this.clusterService = clusterService; this.repositoriesService = repositoriesService; @@ -137,7 +137,7 @@ protected void doExecute(Task task, Request request, ActionListener li if (request.getReroutedFrom() != null) { assert false : request.getReroutedFrom(); throw new IllegalArgumentException( - "speed test of repository [" + "analysis of repository [" + request.getRepositoryName() + "] rerouted from [" + request.getReroutedFrom() @@ -149,16 +149,14 @@ protected void doExecute(Task task, Request request, ActionListener li final List snapshotNodes = getSnapshotNodes(state.nodes()); if (snapshotNodes.isEmpty()) { listener.onFailure( - new IllegalArgumentException( - "no snapshot nodes found for speed test of repository [" + request.getRepositoryName() + "]" - ) + new IllegalArgumentException("no snapshot nodes found for analysis of repository [" + request.getRepositoryName() + "]") ); } else { if (snapshotNodes.size() > 1) { snapshotNodes.remove(state.nodes().getMasterNode()); } final DiscoveryNode targetNode = snapshotNodes.get(new Random(request.getSeed()).nextInt(snapshotNodes.size())); - RepositorySpeedTestAction.logger.trace("rerouting speed test [{}] to [{}]", request.getDescription(), targetNode); + RepositoryAnalyseAction.logger.trace("rerouting analysis [{}] to [{}]", request.getDescription(), targetNode); transportService.sendChildRequest( targetNode, NAME, @@ -199,7 +197,7 @@ public static class AsyncAction { private final long timeoutTimeMillis; // choose the blob path nondeterministically to avoid clashes, assuming that the actual path doesn't matter for reproduction - private final String blobPath = "temp-speed-test-" + UUIDs.randomBase64UUID(); + private final String blobPath = "temp-analysis-" + UUIDs.randomBase64UUID(); private final Queue queue = ConcurrentCollections.newQueue(); private final AtomicReference failure = new AtomicReference<>(); @@ -207,8 +205,8 @@ public static class AsyncAction { private final int workerCount; private final GroupedActionListener workersListener; private final Set expectedBlobs = ConcurrentCollections.newConcurrentSet(); - private final List responses; - private final SpeedTestSummary.Builder summary = new SpeedTestSummary.Builder(); + private final List responses; + private final RepositoryPerformanceSummary.Builder summary = new RepositoryPerformanceSummary.Builder(); @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private OptionalLong listingStartTimeNanos = OptionalLong.empty(); @@ -287,7 +285,7 @@ private boolean isRunning() { if (timeoutTimeMillis < currentTimeMillisSupplier.getAsLong()) { if (failure.compareAndSet( null, - new RepositoryVerificationException(request.repositoryName, "speed test timed out after [" + request.getTimeout() + "]") + new RepositoryVerificationException(request.repositoryName, "analysis timed out after [" + request.getTimeout() + "]") )) { transportService.getTaskManager().cancelTaskAndDescendants(task, "timed out", false, ActionListener.wrap(() -> {})); } @@ -301,7 +299,7 @@ private boolean isRunning() { public void run() { assert queue.isEmpty() && failure.get() == null : "must only run action once"; - logger.info("running speed test of repository [{}] using path [{}]", request.getRepositoryName(), blobPath); + logger.info("running analysis of repository [{}] using path [{}]", request.getRepositoryName(), blobPath); final Random random = new Random(request.getSeed()); @@ -319,10 +317,10 @@ public void run() { final boolean smallBlob = targetLength <= Integer.MAX_VALUE; // we only use the non-atomic API for larger blobs final VerifyBlobTask verifyBlobTask = new VerifyBlobTask( nodes.get(random.nextInt(nodes.size())), - new BlobSpeedTestAction.Request( + new BlobAnalyseAction.Request( request.getRepositoryName(), blobPath, - "speed-test-" + i + "-" + UUIDs.randomBase64UUID(random), + "test-blob-" + i + "-" + UUIDs.randomBase64UUID(random), targetLength, random.nextLong(), nodes, @@ -353,13 +351,13 @@ private void processNextTask() { ); transportService.sendChildRequest( thisTask.node, - BlobSpeedTestAction.NAME, + BlobAnalyseAction.NAME, thisTask.request, task, transportRequestOptions, - new TransportResponseHandler() { + new TransportResponseHandler() { @Override - public void handleResponse(BlobSpeedTestAction.Response response) { + public void handleResponse(BlobAnalyseAction.Response response) { logger.trace("finished [{}]", thisTask); expectedBlobs.add(thisTask.request.getBlobName()); // each task cleans up its own mess on failure if (request.detailed) { @@ -379,8 +377,8 @@ public void handleException(TransportException exp) { } @Override - public BlobSpeedTestAction.Response read(StreamInput in) throws IOException { - return new BlobSpeedTestAction.Response(in); + public BlobAnalyseAction.Response read(StreamInput in) throws IOException { + return new BlobAnalyseAction.Response(in); } } ); @@ -399,7 +397,7 @@ private void onWorkerCompletion() { private void tryListAndCleanup(final int retryCount) { if (timeoutTimeMillis < currentTimeMillisSupplier.getAsLong() || task.isCancelled()) { logger.warn( - "speed test of repository [{}] failed in cleanup phase after [{}] attempts, attempting best-effort cleanup " + "analysis of repository [{}] failed in cleanup phase after [{}] attempts, attempting best-effort cleanup " + "but you may need to manually remove [{}]", request.getRepositoryName(), retryCount, @@ -517,11 +515,11 @@ private void sendResponse() { ) ); } else { - logger.debug(new ParameterizedMessage("speed test of repository [{}] failed", request.repositoryName), exception); + logger.debug(new ParameterizedMessage("analysis of repository [{}] failed", request.repositoryName), exception); listener.onFailure( new RepositoryVerificationException( request.getRepositoryName(), - "speed test failed, you may need to manually remove [" + blobPath + "]", + "analysis failed, you may need to manually remove [" + blobPath + "]", exception ) ); @@ -530,9 +528,9 @@ private void sendResponse() { private static class VerifyBlobTask { final DiscoveryNode node; - final BlobSpeedTestAction.Request request; + final BlobAnalyseAction.Request request; - VerifyBlobTask(DiscoveryNode node, BlobSpeedTestAction.Request request) { + VerifyBlobTask(DiscoveryNode node, BlobAnalyseAction.Request request) { this.node = node; this.request = request; } @@ -715,7 +713,7 @@ public String toString() { @Override public String getDescription() { - return "speed test [repository=" + return "analysis [repository=" + repositoryName + ", blobCount=" + blobCount @@ -759,8 +757,8 @@ public static class Response extends ActionResponse implements StatusToXContentO private final long seed; private final double rareActionProbability; private final String blobPath; - private final SpeedTestSummary summary; - private final List blobResponses; + private final RepositoryPerformanceSummary summary; + private final List blobResponses; private final long listingTimeNanos; private final long deleteTimeNanos; @@ -776,8 +774,8 @@ public Response( long seed, double rareActionProbability, String blobPath, - SpeedTestSummary summary, - List blobResponses, + RepositoryPerformanceSummary summary, + List blobResponses, long listingTimeNanos, long deleteTimeNanos ) { @@ -811,8 +809,8 @@ public Response(StreamInput in) throws IOException { seed = in.readLong(); rareActionProbability = in.readDouble(); blobPath = in.readString(); - summary = new SpeedTestSummary(in); - blobResponses = in.readList(BlobSpeedTestAction.Response::new); + summary = new RepositoryPerformanceSummary(in); + blobResponses = in.readList(BlobAnalyseAction.Response::new); listingTimeNanos = in.readVLong(); deleteTimeNanos = in.readVLong(); } @@ -851,6 +849,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); builder.field("repository", repositoryName); + + // TODO add "consistency": "no problems found" + + // TODO wrap the speed test results into an inner object + builder.field("blob_count", blobCount); builder.field("concurrency", concurrency); builder.field("read_node_count", readNodeCount); @@ -863,7 +866,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (blobResponses.size() > 0) { builder.startArray("details"); - for (BlobSpeedTestAction.Response blobResponse : blobResponses) { + for (BlobAnalyseAction.Response blobResponse : blobResponses) { blobResponse.toXContent(builder, params); } builder.endArray(); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java similarity index 92% rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java index dca306512bb32..5e1990a403a62 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SpeedTestSummary.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java @@ -17,7 +17,7 @@ import java.util.concurrent.atomic.LongAccumulator; import java.util.concurrent.atomic.LongAdder; -public class SpeedTestSummary implements Writeable, ToXContentFragment { +public class RepositoryPerformanceSummary implements Writeable, ToXContentFragment { private final long writeCount; private final long writeBytes; @@ -30,7 +30,7 @@ public class SpeedTestSummary implements Writeable, ToXContentFragment { private final long readThrottledNanos; private final long readElapsedNanos; - public SpeedTestSummary( + public RepositoryPerformanceSummary( long writeCount, long writeBytes, long writeThrottledNanos, @@ -54,7 +54,7 @@ public SpeedTestSummary( this.readElapsedNanos = readElapsedNanos; } - public SpeedTestSummary(StreamInput in) throws IOException { + public RepositoryPerformanceSummary(StreamInput in) throws IOException { writeCount = in.readVLong(); writeBytes = in.readVLong(); writeThrottledNanos = in.readVLong(); @@ -119,8 +119,8 @@ static class Builder { private final LongAdder readThrottledNanos = new LongAdder(); private final LongAdder readElapsedNanos = new LongAdder(); - public SpeedTestSummary build() { - return new SpeedTestSummary( + public RepositoryPerformanceSummary build() { + return new RepositoryPerformanceSummary( writeCount.longValue(), writeBytes.longValue(), writeThrottledNanos.longValue(), @@ -134,7 +134,7 @@ public SpeedTestSummary build() { ); } - public void add(BlobSpeedTestAction.Response response) { + public void add(BlobAnalyseAction.Response response) { writeCount.add(1L); writeBytes.add(response.getWriteBytes()); writeThrottledNanos.add(response.getWriteThrottledNanos()); @@ -142,7 +142,7 @@ public void add(BlobSpeedTestAction.Response response) { final long checksumBytes = response.getChecksumBytes(); - for (final BlobSpeedTestAction.ReadDetail readDetail : response.getReadDetails()) { + for (final BlobAnalyseAction.ReadDetail readDetail : response.getReadDetails()) { readCount.add(1L); readBytes.add(checksumBytes); readWaitNanos.add(readDetail.getFirstByteNanos()); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java new file mode 100644 index 0000000000000..c7075b152d8bb --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestStatusToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class RestRepositoryAnalyseAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new Route(POST, "/_snapshot/{repository}/_analyse")); + } + + @Override + public String getName() { + return "repository_analyse"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { + RepositoryAnalyseAction.Request analyseRepositoryRequest = new RepositoryAnalyseAction.Request(request.param("repository")); + + analyseRepositoryRequest.blobCount(request.paramAsInt("blob_count", analyseRepositoryRequest.getBlobCount())); + analyseRepositoryRequest.concurrency(request.paramAsInt("concurrency", analyseRepositoryRequest.getConcurrency())); + analyseRepositoryRequest.readNodeCount(request.paramAsInt("read_node_count", analyseRepositoryRequest.getReadNodeCount())); + analyseRepositoryRequest.earlyReadNodeCount( + request.paramAsInt("early_read_node_count", analyseRepositoryRequest.getEarlyReadNodeCount()) + ); + analyseRepositoryRequest.seed(request.paramAsLong("seed", analyseRepositoryRequest.getSeed())); + analyseRepositoryRequest.rareActionProbability( + request.paramAsDouble("rare_action_probability", analyseRepositoryRequest.getRareActionProbability()) + ); + analyseRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", analyseRepositoryRequest.getMaxBlobSize())); + analyseRepositoryRequest.timeout(request.paramAsTime("timeout", analyseRepositoryRequest.getTimeout())); + analyseRepositoryRequest.detailed(request.paramAsBoolean("detailed", analyseRepositoryRequest.getDetailed())); + + RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); + return channel -> cancelClient.execute( + RepositoryAnalyseAction.INSTANCE, + analyseRepositoryRequest, + new RestStatusToXContentListener<>(channel) + ); + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java deleted file mode 100644 index 5375280190ba2..0000000000000 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositorySpeedTestAction.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.repositories.blobstore.testkit; - -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestCancellableNodeClient; -import org.elasticsearch.rest.action.RestStatusToXContentListener; - -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.POST; - -public class RestRepositorySpeedTestAction extends BaseRestHandler { - - @Override - public List routes() { - return List.of(new Route(POST, "/_snapshot/{repository}/_speed_test")); - } - - @Override - public String getName() { - return "repository_speed_test"; - } - - @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { - RepositorySpeedTestAction.Request verifyRepositoryRequest = new RepositorySpeedTestAction.Request(request.param("repository")); - - verifyRepositoryRequest.blobCount(request.paramAsInt("blob_count", verifyRepositoryRequest.getBlobCount())); - verifyRepositoryRequest.concurrency(request.paramAsInt("concurrency", verifyRepositoryRequest.getConcurrency())); - verifyRepositoryRequest.readNodeCount(request.paramAsInt("read_node_count", verifyRepositoryRequest.getReadNodeCount())); - verifyRepositoryRequest.earlyReadNodeCount( - request.paramAsInt("early_read_node_count", verifyRepositoryRequest.getEarlyReadNodeCount()) - ); - verifyRepositoryRequest.seed(request.paramAsLong("seed", verifyRepositoryRequest.getSeed())); - verifyRepositoryRequest.rareActionProbability( - request.paramAsDouble("rare_action_probability", verifyRepositoryRequest.getRareActionProbability()) - ); - verifyRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", verifyRepositoryRequest.getMaxBlobSize())); - verifyRepositoryRequest.timeout(request.paramAsTime("timeout", verifyRepositoryRequest.getTimeout())); - verifyRepositoryRequest.detailed(request.paramAsBoolean("detailed", verifyRepositoryRequest.getDetailed())); - - RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); - return channel -> cancelClient.execute( - RepositorySpeedTestAction.INSTANCE, - verifyRepositoryRequest, - new RestStatusToXContentListener<>(channel) - ); - } -} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java index 0d60f4beea058..cac77bcb56783 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java @@ -28,8 +28,8 @@ public class SnapshotRepositoryTestKit extends Plugin implements ActionPlugin { @Override public List> getActions() { return List.of( - new ActionHandler<>(RepositorySpeedTestAction.INSTANCE, RepositorySpeedTestAction.TransportAction.class), - new ActionHandler<>(BlobSpeedTestAction.INSTANCE, BlobSpeedTestAction.TransportAction.class), + new ActionHandler<>(RepositoryAnalyseAction.INSTANCE, RepositoryAnalyseAction.TransportAction.class), + new ActionHandler<>(BlobAnalyseAction.INSTANCE, BlobAnalyseAction.TransportAction.class), new ActionHandler<>(GetBlobChecksumAction.INSTANCE, GetBlobChecksumAction.TransportAction.class) ); } @@ -44,6 +44,6 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - return List.of(new RestRepositorySpeedTestAction()); + return List.of(new RestRepositoryAnalyseAction()); } } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java index 7c49e0a3f0b29..238f9174b425f 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java @@ -18,7 +18,7 @@ public abstract class AbstractSnapshotRepoTestKitRestTestCase extends ESRestTest protected abstract Settings repositorySettings(); - public void testRepositorySpeedTest() throws Exception { + public void testRepositoryAnalysis() throws Exception { final String repositoryType = repositoryType(); final Settings repositorySettings = repositorySettings(); @@ -26,7 +26,7 @@ public void testRepositorySpeedTest() throws Exception { logger.info("creating repository [{}] of type [{}]", repository, repositoryType); registerRepository(repository, repositoryType, true, repositorySettings); - final Request request = new Request(HttpPost.METHOD_NAME, "/_snapshot/" + repository + "/_speed_test"); + final Request request = new Request(HttpPost.METHOD_NAME, "/_snapshot/" + repository + "/_analyse"); request.addParameter("blob_count", "10"); request.addParameter("concurrency", "4"); request.addParameter("max_blob_size", "1mb"); From 32169f4b0847e069250c6eeb94223474b27579e5 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 09:45:46 +0000 Subject: [PATCH 21/43] Implementation details etc --- .../apis/repo-analysis-api.asciidoc | 123 ++++++++++++++---- 1 file changed, 96 insertions(+), 27 deletions(-) diff --git a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc index 73eba9fb83ea6..04b343f2eafc5 100644 --- a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc @@ -24,7 +24,7 @@ PUT /_snapshot/my_repository [source,console] ---- -POST /_snapshot/my_repository/_analyse?blob_count=10&concurrency=4&max_blob_size=1mb&timeout=120s +POST /_snapshot/my_repository/_analyse?blob_count=10&max_blob_size=1mb&timeout=120s ---- [[repo-analysis-api-request]] @@ -44,21 +44,17 @@ The Repository analysis API performs a collection of read and write operations on your repository which are specially designed to detect incorrect behaviour and to measure the performance characteristics of your storage system. -Each analysis runs a wide variety of operations generated by a pseudo-random -process. You can seed this process using the optional `seed` parameter in order -to repeat the same set of operations in multiple experiments. Note that the -operations are performed concurrently so may not always happen in the same -order on each run. - The default values for the parameters to this API are deliberately low to -reduce the impact of running this API accidentally. A realistic experiment -should set `blob_count` to at least `2000` and `max_blob_size` to at least -`2gb`, and will almost certainly need to increase the `timeout` to allow time -for the process to complete successfully. +reduce the impact of running an analysis. A realistic experiment should set +`blob_count` to at least `2000` and `max_blob_size` to at least `2gb`, and will +almost certainly need to increase the `timeout` to allow time for the process +to complete successfully. You should run the analysis on a multi-node cluster +of a similar size to your production cluster so that it can detect any problems +that only arise when the repository is accessed by many nodes at once. If the analysis is successful this API returns details of the testing process, -including how long each operation took. You can use this information to -determine the performance of your storage system. If any operation fails or +optionally including how long each operation took. You can use this information +to determine the performance of your storage system. If any operation fails or returns an incorrect result, this API returns an error. If the API returns an error then it may not have removed all the data it wrote to the repository. The error will indicate the location of any leftover data, and this path is also @@ -67,7 +63,7 @@ been cleaned up correctly. If there is still leftover data at the specified location then you should manually remove it. If the connection from your client to {es} is closed while the client is -waiting for the result of the analysis then the test is cancelled. Since ab +waiting for the result of the analysis then the test is cancelled. Since an analysis takes a long time to complete, you may need to configure your client to wait for longer than usual for a response. On cancellation the analysis attempts to clean up the data it was writing, but it may not be able to remove @@ -76,15 +72,69 @@ verify yourself that this location has been cleaned up correctly. If there is still leftover data at the specified location then you should manually remove it. -NOTE: An analysis writes a substantial amount of data to your repository and -then reads it back again. This consumes bandwidth on the network between the -cluster and the repository, and storage space and IO bandwidth on the +If the analysis is successful then it detected no incorrect behaviour, but this +does not mean that correct behaviour is guaranteed. The analysis attempts to +detect common bugs but it certainly does not offer 100% coverage. Additionally, +it does not test the following: + +- Your repository must perform durable writes. Once a blob has been written it + must remain in place until it is deleted, even after a power loss or similar + disaster. + +- Your repository must not suffer from silent data corruption. Once a blob has + been written its contents must remain unchanged until it is deliberately + modified or deleted. + +- Your repository must behave correctly even if connectivity from the cluster + is disrupted. Reads and writes may fail in this case, but they must not return + incorrect results. + +IMPORTANT: An analysis writes a substantial amount of data to your repository +and then reads it back again. This consumes bandwidth on the network between +the cluster and the repository, and storage space and IO bandwidth on the repository itself. You must ensure this load does not affect other users of these systems. Analyses respect the repository settings `max_snapshot_bytes_per_sec` and `max_restore_bytes_per_sec` if available, and the cluster setting `indices.recovery.max_bytes_per_sec` which you can use to limit the bandwidth they consume. +==== Implementation details + +NOTE: This section of documentation describes how the Repository analysis API +works in this version of {es}, but you should expect the implementation to vary +between versions. The response format exposes details of the implementation so +it may also be different in newer versions. + +The analysis comprises a number of blob-level tasks, as set by the `blob_count` +parameter. The blob-level tasks are distributed over the data and +master-eligible nodes in the cluster for execution. + +For most blob-level tasks, the executing node first writes a blob to the +repository, and then instructs some of the other nodes in the cluster to +attempt to read the data it just wrote. The size of the blob is chosen +randomly, according to the `max_blob_size` parameter. If any of these reads +fails then the repository does not implement the necessary read-after-write +semantics that {es} requires. + +For some blob-level tasks, the executing node will instruct some of its peers +to attempt to read the data before the writing process completes. These reads +are permitted to fail, but must not return partial data. If any read returns +partial data then the repository does not implement the necessary atomicity +semantics that {es} requires. + +For some blob-level tasks, the executing node will overwrite the blob while its +peers are reading it. In this case the data read may come from either the +original or the overwritten blob, but must not return partial data or a mix of +data from the two blobs. If any of these reads returns partial data or a mix of +the two blobs then the repository does not implement the necessary atomicity +semantics that {es} requires for overwrites. + +The executing node will use a variety of different methods to write the blob. +For instance, where applicable, it will use both single-part and multi-part +uploads. Similarly, the reading nodes will use a variety of different methods +to read the data back again. For instance they may read the entire blob from +start to end, or may read only a subset of the data. + [[repo-analysis-api-path-params]] ==== {api-path-parms-title} @@ -100,6 +150,21 @@ Name of the snapshot repository to test. the test. Defaults to `100`. For realistic experiments you should set this to at least `2000`. +`max_blob_size`:: +(Optional, <>) The maximum size of a blob to be written +during the test. Defaults to `10mb`. For realistic experiments you should set +this to at least `2gb`. + +`timeout`:: +(Optional, <>) Specifies the period of time to wait for +the test to complete. If no response is received before the timeout expires, +the test is cancelled and returns an error. Defaults to `30s`. + +===== Advanced query parameters + +The following parameters allow additional control over the analysis, but you +will usually not need to adjust them. + `concurrency`:: (Optional, integer) The number of write operations to perform concurrently. Defaults to `10`. @@ -113,26 +178,30 @@ after writing each blob. Defaults to `10`. operation while writing each blob. Defaults to `2`. Early read operations are only rarely performed. -`max_blob_size`:: -(Optional, <>) The maximum size of a blob to be written -during the test. Defaults to `10mb`. For realistic experiments you should set -this to at least `2gb`. +`rare_action_probability`:: +(Optional, double) The probability of performing a rare action (an early read +or an overwrite) on each blob. Defaults to `0.02`. `seed`:: (Optional, integer) The seed for the pseudo-random number generator used to generate the list of operations performed during the test. To repeat the same set of operations in multiple experiments, use the same seed in each -experiment. +experiment. Note that the operations are performed concurrently so may not +always happen in the same order on each run. -`timeout`:: -(Optional, <>) Specifies the period of time to wait for -the test to complete. If no response is received before the timeout expires, -the test is cancelled and returns an error. Defaults to `30s`. +`detailed`:: +(Optional, boolean) Whether to return detailed results, including timing +information for every operation performed during the analysis. Defaults to +`false`, meaning to return only a summary of the analysis. [role="child_attributes"] [[repo-analysis-api-response-body]] ==== {api-response-body-title} +The response exposes implementation details of the analysis which may change +from version to version. The response body format is therefore not considered +stable and may be different in newer versions. + `coordinating_node`:: (object) Identifies the node which coordinated the analysis and performed the final cleanup. @@ -267,7 +336,7 @@ The total elapsed time spent on reading blobs in the test, in nanoseconds. `details`:: (array) A description of every read and write operation performed during the test. This is -only returned if the `?details` request parameter is set to `true`. +only returned if the `?detailed` request parameter is set to `true`. + .Properties of items within `details` [%collapsible] From 452968a566eff1b823893f9075d7878a07566231 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 10:08:27 +0000 Subject: [PATCH 22/43] Add TODO for writeBlobRandomly --- .../repositories/blobstore/testkit/BlobAnalyseAction.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java index f6960e38b41b3..433609c770a00 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java @@ -287,6 +287,11 @@ private void writeRandomBlob(boolean atomic, boolean failIfExists, Runnable onLa } final long startNanos = System.nanoTime(); ActionListener.completeWith(stepListener, () -> { + + // TODO push some of this writing logic down into the blob container implementation. + // E.g. for S3 blob containers we would like to choose somewhat more randomly between single-part and multi-part uploads, + // rather than relying on the usual distinction based on the size of the blob. + if (atomic || (request.targetLength <= Integer.MAX_VALUE && random.nextBoolean())) { final RandomBlobContentBytesReference bytesReference = new RandomBlobContentBytesReference( content, From 6076d346c7546f5830f74af222905fead727d2e9 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 10:09:42 +0000 Subject: [PATCH 23/43] Add TODO for delete verification --- .../repositories/blobstore/testkit/RepositoryAnalyseAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java index a542d615a24f7..613e98ae61001 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java @@ -479,6 +479,7 @@ private void deleteContainerAndSendResponse() { assert deleteStartTimeNanos.isEmpty() : deleteStartTimeNanos; deleteStartTimeNanos = OptionalLong.of(System.nanoTime()); getBlobContainer().delete(); + // TODO can we check that the delete actually succeeded here? } catch (Exception e) { fail(e); } finally { From a50c0d17d5e194c4c97eeb2a58645319a38c6e27 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 10:10:47 +0000 Subject: [PATCH 24/43] Add TODO to remove cleanup retries --- .../repositories/blobstore/testkit/RepositoryAnalyseAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java index 613e98ae61001..66ab62236e133 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java @@ -395,6 +395,7 @@ private void onWorkerCompletion() { } private void tryListAndCleanup(final int retryCount) { + // TODO no need for retries any more now that S3 has consistent listings if (timeoutTimeMillis < currentTimeMillisSupplier.getAsLong() || task.isCancelled()) { logger.warn( "analysis of repository [{}] failed in cleanup phase after [{}] attempts, attempting best-effort cleanup " From 4b6d53b561af76583fc69e03ffb2123fe7896bad Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 11:07:17 +0000 Subject: [PATCH 25/43] Add plumbing for max_total_data_size --- .../apis/repo-analysis-api.asciidoc | 27 +++++++++++++------ .../api/snapshot.repository_analyse.json | 4 +++ .../rest-api-spec/test/10_analyse.yml | 2 ++ .../testkit/RepositoryAnalyseAction.java | 24 +++++++++++++++++ .../testkit/RestRepositoryAnalyseAction.java | 3 +++ 5 files changed, 52 insertions(+), 8 deletions(-) diff --git a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc index 04b343f2eafc5..62238fffcd035 100644 --- a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc @@ -46,11 +46,12 @@ and to measure the performance characteristics of your storage system. The default values for the parameters to this API are deliberately low to reduce the impact of running an analysis. A realistic experiment should set -`blob_count` to at least `2000` and `max_blob_size` to at least `2gb`, and will -almost certainly need to increase the `timeout` to allow time for the process -to complete successfully. You should run the analysis on a multi-node cluster -of a similar size to your production cluster so that it can detect any problems -that only arise when the repository is accessed by many nodes at once. +`blob_count` to at least `2000`, `max_blob_size` to at least `2gb`, and +`max_total_data_size` to at least `1tb`, and will almost certainly need to +increase the `timeout` to allow time for the process to complete successfully. +You should run the analysis on a multi-node cluster of a similar size to your +production cluster so that it can detect any problems that only arise when the +repository is accessed by many nodes at once. If the analysis is successful this API returns details of the testing process, optionally including how long each operation took. You can use this information @@ -112,9 +113,9 @@ master-eligible nodes in the cluster for execution. For most blob-level tasks, the executing node first writes a blob to the repository, and then instructs some of the other nodes in the cluster to attempt to read the data it just wrote. The size of the blob is chosen -randomly, according to the `max_blob_size` parameter. If any of these reads -fails then the repository does not implement the necessary read-after-write -semantics that {es} requires. +randomly, according to the `max_blob_size` and `max_total_data_size` +parameters. If any of these reads fails then the repository does not implement +the necessary read-after-write semantics that {es} requires. For some blob-level tasks, the executing node will instruct some of its peers to attempt to read the data before the writing process completes. These reads @@ -155,6 +156,11 @@ at least `2000`. during the test. Defaults to `10mb`. For realistic experiments you should set this to at least `2gb`. +`max_total_data_size`:: +(Optional, <>) An upper limit on the total size of all +the blobs written during the test. Defaults to `1gb`. For realistic experiments +you should set this to at least `1tb`. + `timeout`:: (Optional, <>) Specifies the period of time to wait for the test to complete. If no response is received before the timeout expires, @@ -248,6 +254,11 @@ parameter. The limit on the size of a blob written during the test, equal to the `?max_blob_size` parameter. +`max_total_data_size`:: +(string) +The limit on the total size of all blob written during the test, equal to the +`?max_total_data_size` parameter. + `seed`:: (long) The seed for the pseudo-random number generator used to generate the operations diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json index 5439d38995ee3..bbdf9ca343063 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json @@ -51,6 +51,10 @@ "type":"string", "description":"Maximum size of a blob to create during the test, e.g '1gb' or '100mb'. Defaults to '10mb'." }, + "max_total_data_size":{ + "type":"string", + "description":"Maximum total size of all blobs to create during the test, e.g '1tb' or '100gb'. Defaults to '1gb'." + }, "timeout":{ "type":"time", "description":"Explicit operation timeout. Defaults to '30s'." diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml index cc1d8b95700da..234be07549dc4 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml @@ -49,6 +49,7 @@ setup: read_node_count: 2 early_read_node_count: 1 rare_action_probability: 0.01 + max_total_data_size: 5mb - is_true: coordinating_node.id - is_true: coordinating_node.name @@ -59,6 +60,7 @@ setup: - match: { early_read_node_count: 1 } - match: { rare_action_probability: 0.01 } - match: { max_blob_size: 1mb } + - match: { max_total_data_size: 5mb } - is_true: seed - is_true: blob_path - is_false: details diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java index 66ab62236e133..68c74ceede3be 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyseAction.java @@ -312,6 +312,7 @@ public void run() { } blobSizes.add(maxBlobSize); + // TODO account for max total blob size for (int i = 0; i < request.getBlobCount(); i++) { final long targetLength = blobSizes.get(random.nextInt(blobSizes.size())); final boolean smallBlob = targetLength <= Integer.MAX_VALUE; // we only use the non-atomic API for larger blobs @@ -507,6 +508,7 @@ private void sendResponse() { request.readNodeCount, request.earlyReadNodeCount, request.maxBlobSize, + request.maxTotalDataSize, request.seed, request.rareActionProbability, blobPath, @@ -556,6 +558,7 @@ public static class Request extends ActionRequest { private double rareActionProbability = 0.02; private TimeValue timeout = TimeValue.timeValueSeconds(30); private ByteSizeValue maxBlobSize = ByteSizeValue.ofMb(10); + private ByteSizeValue maxTotalDataSize = ByteSizeValue.ofGb(1); private boolean detailed = false; private DiscoveryNode reroutedFrom = null; @@ -574,6 +577,7 @@ public Request(StreamInput in) throws IOException { earlyReadNodeCount = in.readVInt(); timeout = in.readTimeValue(); maxBlobSize = new ByteSizeValue(in); + maxTotalDataSize = new ByteSizeValue(in); detailed = in.readBoolean(); reroutedFrom = in.readOptionalWriteable(DiscoveryNode::new); } @@ -595,6 +599,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(earlyReadNodeCount); out.writeTimeValue(timeout); maxBlobSize.writeTo(out); + maxTotalDataSize.writeTo(out); out.writeBoolean(detailed); out.writeOptionalWriteable(reroutedFrom); } @@ -633,6 +638,13 @@ public void maxBlobSize(ByteSizeValue maxBlobSize) { this.maxBlobSize = maxBlobSize; } + public void maxTotalDataSize(ByteSizeValue maxTotalDataSize) { + if (maxTotalDataSize.getBytes() <= 0) { + throw new IllegalArgumentException("maxTotalDataSize must be >0, but was [" + maxTotalDataSize + "]"); + } + this.maxTotalDataSize = maxTotalDataSize; + } + public void detailed(boolean detailed) { this.detailed = detailed; } @@ -661,6 +673,10 @@ public ByteSizeValue getMaxBlobSize() { return maxBlobSize; } + public ByteSizeValue getMaxTotalDataSize() { + return maxTotalDataSize; + } + public boolean getDetailed() { return detailed; } @@ -733,6 +749,8 @@ public String getDescription() { + timeout + ", maxBlobSize=" + maxBlobSize + + ", maxTotalDataSize=" + + maxTotalDataSize + ", detailed=" + detailed + "]"; @@ -756,6 +774,7 @@ public static class Response extends ActionResponse implements StatusToXContentO private final int readNodeCount; private final int earlyReadNodeCount; private final ByteSizeValue maxBlobSize; + private final ByteSizeValue maxTotalDataSize; private final long seed; private final double rareActionProbability; private final String blobPath; @@ -773,6 +792,7 @@ public Response( int readNodeCount, int earlyReadNodeCount, ByteSizeValue maxBlobSize, + ByteSizeValue maxTotalDataSize, long seed, double rareActionProbability, String blobPath, @@ -789,6 +809,7 @@ public Response( this.readNodeCount = readNodeCount; this.earlyReadNodeCount = earlyReadNodeCount; this.maxBlobSize = maxBlobSize; + this.maxTotalDataSize = maxTotalDataSize; this.seed = seed; this.rareActionProbability = rareActionProbability; this.blobPath = blobPath; @@ -808,6 +829,7 @@ public Response(StreamInput in) throws IOException { readNodeCount = in.readVInt(); earlyReadNodeCount = in.readVInt(); maxBlobSize = new ByteSizeValue(in); + maxTotalDataSize = new ByteSizeValue(in); seed = in.readLong(); rareActionProbability = in.readDouble(); blobPath = in.readString(); @@ -827,6 +849,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(readNodeCount); out.writeVInt(earlyReadNodeCount); maxBlobSize.writeTo(out); + maxTotalDataSize.writeTo(out); out.writeLong(seed); out.writeDouble(rareActionProbability); out.writeString(blobPath); @@ -861,6 +884,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("read_node_count", readNodeCount); builder.field("early_read_node_count", earlyReadNodeCount); builder.field("max_blob_size", maxBlobSize); + builder.field("max_total_data_size", maxTotalDataSize); builder.field("seed", seed); builder.field("rare_action_probability", rareActionProbability); builder.field("blob_path", blobPath); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java index c7075b152d8bb..fe2b81148323c 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java @@ -44,6 +44,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC request.paramAsDouble("rare_action_probability", analyseRepositoryRequest.getRareActionProbability()) ); analyseRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", analyseRepositoryRequest.getMaxBlobSize())); + analyseRepositoryRequest.maxTotalDataSize( + request.paramAsSize("max_total_data_size", analyseRepositoryRequest.getMaxTotalDataSize()) + ); analyseRepositoryRequest.timeout(request.paramAsTime("timeout", analyseRepositoryRequest.getTimeout())); analyseRepositoryRequest.detailed(request.paramAsBoolean("detailed", analyseRepositoryRequest.getDetailed())); From 2263d9f2730279e865c8f59a9fdd6193b8a5b6dd Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 11:07:45 +0000 Subject: [PATCH 26/43] Skip verification, it doesn't work --- .../repositories/blobstore/testkit/RepositoryAnalysisIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java index ad990f6979aea..df4cb18ea3f4d 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java @@ -67,7 +67,7 @@ protected Collection> nodePlugins() { public void testFoo() { - createRepository("test-repo", TestPlugin.DISRUPTABLE_REPO_TYPE); + createRepositoryNoVerify("test-repo", TestPlugin.DISRUPTABLE_REPO_TYPE); final DisruptableBlobStore blobStore = new DisruptableBlobStore(); for (final RepositoriesService repositoriesService : internalCluster().getInstances(RepositoriesService.class)) { From 6662e1c6a5bcc2e3fa50158a961ec4430631bf4e Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 11:14:22 +0000 Subject: [PATCH 27/43] prefer US spelling --- .../apis/repo-analysis-api.asciidoc | 6 +- .../operator/OperatorOnlyRegistry.java | 6 +- ....json => snapshot.repository_analyze.json} | 6 +- .../test/{10_analyse.yml => 10_analyze.yml} | 6 +- .../testkit/RepositoryAnalysisIT.java | 10 ++-- ...lyseAction.java => BlobAnalyzeAction.java} | 22 +++---- .../testkit/GetBlobChecksumAction.java | 2 +- ...tion.java => RepositoryAnalyzeAction.java} | 42 ++++++------- .../testkit/RepositoryPerformanceSummary.java | 4 +- .../testkit/RestRepositoryAnalyseAction.java | 60 ------------------- .../testkit/RestRepositoryAnalyzeAction.java | 60 +++++++++++++++++++ .../testkit/SnapshotRepositoryTestKit.java | 6 +- ...stractSnapshotRepoTestKitRestTestCase.java | 2 +- 13 files changed, 116 insertions(+), 116 deletions(-) rename x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/{snapshot.repository_analyse.json => snapshot.repository_analyze.json} (93%) rename x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/{10_analyse.yml => 10_analyze.yml} (96%) rename x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/{BlobAnalyseAction.java => BlobAnalyzeAction.java} (98%) rename x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/{RepositoryAnalyseAction.java => RepositoryAnalyzeAction.java} (96%) delete mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java diff --git a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc index 62238fffcd035..dd20875dcf031 100644 --- a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc @@ -5,7 +5,7 @@ Repository analysis ++++ -Analyses a repository, reporting its performance characteristics and any +Analyzes a repository, reporting its performance characteristics and any incorrect behaviour found. //// @@ -24,13 +24,13 @@ PUT /_snapshot/my_repository [source,console] ---- -POST /_snapshot/my_repository/_analyse?blob_count=10&max_blob_size=1mb&timeout=120s +POST /_snapshot/my_repository/_analyze?blob_count=10&max_blob_size=1mb&timeout=120s ---- [[repo-analysis-api-request]] ==== {api-request-title} -`POST /_snapshot//_analyse` +`POST /_snapshot//_analyze` [[repo-analysis-api-desc]] ==== {api-description-title} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java index 744755f1e699f..57b2ab699703f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/operator/OperatorOnlyRegistry.java @@ -35,9 +35,9 @@ public class OperatorOnlyRegistry { "cluster:admin/autoscaling/get_autoscaling_policy", "cluster:admin/autoscaling/get_autoscaling_capacity", // Repository analysis actions are not mentioned in core, literal strings are needed. - "cluster:admin/repository/analyse", - "cluster:admin/repository/analyse/blob", - "cluster:admin/repository/analyse/blob/read" + "cluster:admin/repository/analyze", + "cluster:admin/repository/analyze/blob", + "cluster:admin/repository/analyze/blob/read" ); private final ClusterSettings clusterSettings; diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyze.json similarity index 93% rename from x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json rename to x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyze.json index bbdf9ca343063..4c62d0bb52bb4 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyse.json +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/api/snapshot.repository_analyze.json @@ -1,15 +1,15 @@ { - "snapshot.repository_analyse":{ + "snapshot.repository_analyze":{ "documentation":{ "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html", - "description":"Analyses a repository for correctness and performance" + "description":"Analyzes a repository for correctness and performance" }, "stability":"stable", "visibility":"public", "url":{ "paths":[ { - "path":"/_snapshot/{repository}/_analyse", + "path":"/_snapshot/{repository}/_analyze", "methods":[ "POST" ], diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyze.yml similarity index 96% rename from x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml rename to x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyze.yml index 234be07549dc4..18b30eeecf12d 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyse.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyze.yml @@ -26,7 +26,7 @@ setup: - do: catch: bad_request - snapshot.repository_analyse: + snapshot.repository_analyze: repository: test_repo_readonly - match: { status: 400 } @@ -41,7 +41,7 @@ setup: reason: "introduced in 8.0" - do: - snapshot.repository_analyse: + snapshot.repository_analyze: repository: test_repo blob_count: 10 concurrency: 5 @@ -84,7 +84,7 @@ setup: reason: "introduced in 8.0" - do: - snapshot.repository_analyse: + snapshot.repository_analyze: repository: test_repo blob_count: 10 concurrency: 5 diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java index df4cb18ea3f4d..7f20f45222fbe 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java @@ -78,7 +78,7 @@ public void testFoo() { } } - final RepositoryAnalyseAction.Request request = new RepositoryAnalyseAction.Request("test-repo"); + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); if (randomBoolean()) { request.concurrency(between(1, 5)); @@ -98,7 +98,7 @@ public void testFoo() { request.timeout(TimeValue.timeValueSeconds(5)); - final RepositoryAnalyseAction.Response response = client().execute(RepositoryAnalyseAction.INSTANCE, request).actionGet(); + final RepositoryAnalyzeAction.Response response = client().execute(RepositoryAnalyzeAction.INSTANCE, request).actionGet(); assertThat(response.status(), equalTo(RestStatus.OK)); } @@ -164,9 +164,9 @@ public String startVerification() { static class DisruptableBlobStore implements BlobStore { private final Map blobContainers = ConcurrentCollections.newConcurrentMap(); - private Semaphore writeSemaphore = new Semaphore(new RepositoryAnalyseAction.Request("dummy").getConcurrency()); - private int maxBlobCount = new RepositoryAnalyseAction.Request("dummy").getBlobCount(); - private long maxBlobSize = new RepositoryAnalyseAction.Request("dummy").getMaxBlobSize().getBytes(); + private Semaphore writeSemaphore = new Semaphore(new RepositoryAnalyzeAction.Request("dummy").getConcurrency()); + private int maxBlobCount = new RepositoryAnalyzeAction.Request("dummy").getBlobCount(); + private long maxBlobSize = new RepositoryAnalyzeAction.Request("dummy").getMaxBlobSize().getBytes(); @Override public BlobContainer blobContainer(BlobPath path) { diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java similarity index 98% rename from x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java rename to x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java index 433609c770a00..bd6b5233055b5 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyseAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java @@ -140,20 +140,20 @@ * unnecessary resources. * */ -public class BlobAnalyseAction extends ActionType { +public class BlobAnalyzeAction extends ActionType { - private static final Logger logger = LogManager.getLogger(BlobAnalyseAction.class); + private static final Logger logger = LogManager.getLogger(BlobAnalyzeAction.class); - public static final BlobAnalyseAction INSTANCE = new BlobAnalyseAction(); - public static final String NAME = "cluster:admin/repository/analyse/blob"; + public static final BlobAnalyzeAction INSTANCE = new BlobAnalyzeAction(); + public static final String NAME = "cluster:admin/repository/analyze/blob"; - private BlobAnalyseAction() { + private BlobAnalyzeAction() { super(NAME, Response::new); } public static class TransportAction extends HandledTransportAction { - private static final Logger logger = BlobAnalyseAction.logger; + private static final Logger logger = BlobAnalyzeAction.logger; private final RepositoriesService repositoriesService; private final TransportService transportService; @@ -190,10 +190,10 @@ protected void doExecute(Task task, Request request, ActionListener li static class BlobAnalysis { private final TransportService transportService; private final CancellableTask task; - private final BlobAnalyseAction.Request request; + private final BlobAnalyzeAction.Request request; private final BlobStoreRepository repository; private final BlobContainer blobContainer; - private final ActionListener listener; + private final ActionListener listener; private final Random random; private final boolean checksumWholeBlob; private final long checksumStart; @@ -207,10 +207,10 @@ static class BlobAnalysis { BlobAnalysis( TransportService transportService, CancellableTask task, - BlobAnalyseAction.Request request, + BlobAnalyzeAction.Request request, BlobStoreRepository repository, BlobContainer blobContainer, - ActionListener listener + ActionListener listener ) { this.transportService = transportService; this.task = task; @@ -677,7 +677,7 @@ public String getDescription() { @Override public String toString() { - return "BlobAnalyseAction.Request{" + getDescription() + "}"; + return "BlobAnalyzeAction.Request{" + getDescription() + "}"; } @Override diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java index d843e46ecfca5..110647fa86b07 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java @@ -52,7 +52,7 @@ public class GetBlobChecksumAction extends ActionType { +public class RepositoryAnalyzeAction extends ActionType { - private static final Logger logger = LogManager.getLogger(RepositoryAnalyseAction.class); + private static final Logger logger = LogManager.getLogger(RepositoryAnalyzeAction.class); - public static final RepositoryAnalyseAction INSTANCE = new RepositoryAnalyseAction(); - public static final String NAME = "cluster:admin/repository/analyse"; + public static final RepositoryAnalyzeAction INSTANCE = new RepositoryAnalyzeAction(); + public static final String NAME = "cluster:admin/repository/analyze"; - private RepositoryAnalyseAction() { + private RepositoryAnalyzeAction() { super(NAME, Response::new); } @@ -98,7 +98,7 @@ public TransportAction( ClusterService clusterService, RepositoriesService repositoriesService ) { - super(NAME, transportService, actionFilters, RepositoryAnalyseAction.Request::new, ThreadPool.Names.SAME); + super(NAME, transportService, actionFilters, RepositoryAnalyzeAction.Request::new, ThreadPool.Names.SAME); this.transportService = transportService; this.clusterService = clusterService; this.repositoriesService = repositoriesService; @@ -156,7 +156,7 @@ protected void doExecute(Task task, Request request, ActionListener li snapshotNodes.remove(state.nodes().getMasterNode()); } final DiscoveryNode targetNode = snapshotNodes.get(new Random(request.getSeed()).nextInt(snapshotNodes.size())); - RepositoryAnalyseAction.logger.trace("rerouting analysis [{}] to [{}]", request.getDescription(), targetNode); + RepositoryAnalyzeAction.logger.trace("rerouting analysis [{}] to [{}]", request.getDescription(), targetNode); transportService.sendChildRequest( targetNode, NAME, @@ -205,7 +205,7 @@ public static class AsyncAction { private final int workerCount; private final GroupedActionListener workersListener; private final Set expectedBlobs = ConcurrentCollections.newConcurrentSet(); - private final List responses; + private final List responses; private final RepositoryPerformanceSummary.Builder summary = new RepositoryPerformanceSummary.Builder(); @SuppressWarnings("OptionalUsedAsFieldOrParameterType") @@ -318,7 +318,7 @@ public void run() { final boolean smallBlob = targetLength <= Integer.MAX_VALUE; // we only use the non-atomic API for larger blobs final VerifyBlobTask verifyBlobTask = new VerifyBlobTask( nodes.get(random.nextInt(nodes.size())), - new BlobAnalyseAction.Request( + new BlobAnalyzeAction.Request( request.getRepositoryName(), blobPath, "test-blob-" + i + "-" + UUIDs.randomBase64UUID(random), @@ -352,13 +352,13 @@ private void processNextTask() { ); transportService.sendChildRequest( thisTask.node, - BlobAnalyseAction.NAME, + BlobAnalyzeAction.NAME, thisTask.request, task, transportRequestOptions, - new TransportResponseHandler() { + new TransportResponseHandler() { @Override - public void handleResponse(BlobAnalyseAction.Response response) { + public void handleResponse(BlobAnalyzeAction.Response response) { logger.trace("finished [{}]", thisTask); expectedBlobs.add(thisTask.request.getBlobName()); // each task cleans up its own mess on failure if (request.detailed) { @@ -378,8 +378,8 @@ public void handleException(TransportException exp) { } @Override - public BlobAnalyseAction.Response read(StreamInput in) throws IOException { - return new BlobAnalyseAction.Response(in); + public BlobAnalyzeAction.Response read(StreamInput in) throws IOException { + return new BlobAnalyzeAction.Response(in); } } ); @@ -532,9 +532,9 @@ private void sendResponse() { private static class VerifyBlobTask { final DiscoveryNode node; - final BlobAnalyseAction.Request request; + final BlobAnalyzeAction.Request request; - VerifyBlobTask(DiscoveryNode node, BlobAnalyseAction.Request request) { + VerifyBlobTask(DiscoveryNode node, BlobAnalyzeAction.Request request) { this.node = node; this.request = request; } @@ -779,7 +779,7 @@ public static class Response extends ActionResponse implements StatusToXContentO private final double rareActionProbability; private final String blobPath; private final RepositoryPerformanceSummary summary; - private final List blobResponses; + private final List blobResponses; private final long listingTimeNanos; private final long deleteTimeNanos; @@ -797,7 +797,7 @@ public Response( double rareActionProbability, String blobPath, RepositoryPerformanceSummary summary, - List blobResponses, + List blobResponses, long listingTimeNanos, long deleteTimeNanos ) { @@ -834,7 +834,7 @@ public Response(StreamInput in) throws IOException { rareActionProbability = in.readDouble(); blobPath = in.readString(); summary = new RepositoryPerformanceSummary(in); - blobResponses = in.readList(BlobAnalyseAction.Response::new); + blobResponses = in.readList(BlobAnalyzeAction.Response::new); listingTimeNanos = in.readVLong(); deleteTimeNanos = in.readVLong(); } @@ -892,7 +892,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (blobResponses.size() > 0) { builder.startArray("details"); - for (BlobAnalyseAction.Response blobResponse : blobResponses) { + for (BlobAnalyzeAction.Response blobResponse : blobResponses) { blobResponse.toXContent(builder, params); } builder.endArray(); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java index 5e1990a403a62..9459971b29b1f 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java @@ -134,7 +134,7 @@ public RepositoryPerformanceSummary build() { ); } - public void add(BlobAnalyseAction.Response response) { + public void add(BlobAnalyzeAction.Response response) { writeCount.add(1L); writeBytes.add(response.getWriteBytes()); writeThrottledNanos.add(response.getWriteThrottledNanos()); @@ -142,7 +142,7 @@ public void add(BlobAnalyseAction.Response response) { final long checksumBytes = response.getChecksumBytes(); - for (final BlobAnalyseAction.ReadDetail readDetail : response.getReadDetails()) { + for (final BlobAnalyzeAction.ReadDetail readDetail : response.getReadDetails()) { readCount.add(1L); readBytes.add(checksumBytes); readWaitNanos.add(readDetail.getFirstByteNanos()); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java deleted file mode 100644 index fe2b81148323c..0000000000000 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyseAction.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.repositories.blobstore.testkit; - -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.action.RestCancellableNodeClient; -import org.elasticsearch.rest.action.RestStatusToXContentListener; - -import java.util.List; - -import static org.elasticsearch.rest.RestRequest.Method.POST; - -public class RestRepositoryAnalyseAction extends BaseRestHandler { - - @Override - public List routes() { - return List.of(new Route(POST, "/_snapshot/{repository}/_analyse")); - } - - @Override - public String getName() { - return "repository_analyse"; - } - - @Override - public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { - RepositoryAnalyseAction.Request analyseRepositoryRequest = new RepositoryAnalyseAction.Request(request.param("repository")); - - analyseRepositoryRequest.blobCount(request.paramAsInt("blob_count", analyseRepositoryRequest.getBlobCount())); - analyseRepositoryRequest.concurrency(request.paramAsInt("concurrency", analyseRepositoryRequest.getConcurrency())); - analyseRepositoryRequest.readNodeCount(request.paramAsInt("read_node_count", analyseRepositoryRequest.getReadNodeCount())); - analyseRepositoryRequest.earlyReadNodeCount( - request.paramAsInt("early_read_node_count", analyseRepositoryRequest.getEarlyReadNodeCount()) - ); - analyseRepositoryRequest.seed(request.paramAsLong("seed", analyseRepositoryRequest.getSeed())); - analyseRepositoryRequest.rareActionProbability( - request.paramAsDouble("rare_action_probability", analyseRepositoryRequest.getRareActionProbability()) - ); - analyseRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", analyseRepositoryRequest.getMaxBlobSize())); - analyseRepositoryRequest.maxTotalDataSize( - request.paramAsSize("max_total_data_size", analyseRepositoryRequest.getMaxTotalDataSize()) - ); - analyseRepositoryRequest.timeout(request.paramAsTime("timeout", analyseRepositoryRequest.getTimeout())); - analyseRepositoryRequest.detailed(request.paramAsBoolean("detailed", analyseRepositoryRequest.getDetailed())); - - RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); - return channel -> cancelClient.execute( - RepositoryAnalyseAction.INSTANCE, - analyseRepositoryRequest, - new RestStatusToXContentListener<>(channel) - ); - } -} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java new file mode 100644 index 0000000000000..d695eb7c97aa0 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestStatusToXContentListener; + +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class RestRepositoryAnalyzeAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new Route(POST, "/_snapshot/{repository}/_analyze")); + } + + @Override + public String getName() { + return "repository_analyze"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) { + RepositoryAnalyzeAction.Request analyzeRepositoryRequest = new RepositoryAnalyzeAction.Request(request.param("repository")); + + analyzeRepositoryRequest.blobCount(request.paramAsInt("blob_count", analyzeRepositoryRequest.getBlobCount())); + analyzeRepositoryRequest.concurrency(request.paramAsInt("concurrency", analyzeRepositoryRequest.getConcurrency())); + analyzeRepositoryRequest.readNodeCount(request.paramAsInt("read_node_count", analyzeRepositoryRequest.getReadNodeCount())); + analyzeRepositoryRequest.earlyReadNodeCount( + request.paramAsInt("early_read_node_count", analyzeRepositoryRequest.getEarlyReadNodeCount()) + ); + analyzeRepositoryRequest.seed(request.paramAsLong("seed", analyzeRepositoryRequest.getSeed())); + analyzeRepositoryRequest.rareActionProbability( + request.paramAsDouble("rare_action_probability", analyzeRepositoryRequest.getRareActionProbability()) + ); + analyzeRepositoryRequest.maxBlobSize(request.paramAsSize("max_blob_size", analyzeRepositoryRequest.getMaxBlobSize())); + analyzeRepositoryRequest.maxTotalDataSize( + request.paramAsSize("max_total_data_size", analyzeRepositoryRequest.getMaxTotalDataSize()) + ); + analyzeRepositoryRequest.timeout(request.paramAsTime("timeout", analyzeRepositoryRequest.getTimeout())); + analyzeRepositoryRequest.detailed(request.paramAsBoolean("detailed", analyzeRepositoryRequest.getDetailed())); + + RestCancellableNodeClient cancelClient = new RestCancellableNodeClient(client, request.getHttpChannel()); + return channel -> cancelClient.execute( + RepositoryAnalyzeAction.INSTANCE, + analyzeRepositoryRequest, + new RestStatusToXContentListener<>(channel) + ); + } +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java index cac77bcb56783..ffc84078893aa 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java @@ -28,8 +28,8 @@ public class SnapshotRepositoryTestKit extends Plugin implements ActionPlugin { @Override public List> getActions() { return List.of( - new ActionHandler<>(RepositoryAnalyseAction.INSTANCE, RepositoryAnalyseAction.TransportAction.class), - new ActionHandler<>(BlobAnalyseAction.INSTANCE, BlobAnalyseAction.TransportAction.class), + new ActionHandler<>(RepositoryAnalyzeAction.INSTANCE, RepositoryAnalyzeAction.TransportAction.class), + new ActionHandler<>(BlobAnalyzeAction.INSTANCE, BlobAnalyzeAction.TransportAction.class), new ActionHandler<>(GetBlobChecksumAction.INSTANCE, GetBlobChecksumAction.TransportAction.class) ); } @@ -44,6 +44,6 @@ public List getRestHandlers( IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster ) { - return List.of(new RestRepositoryAnalyseAction()); + return List.of(new RestRepositoryAnalyzeAction()); } } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java index 238f9174b425f..c5d506528ae12 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AbstractSnapshotRepoTestKitRestTestCase.java @@ -26,7 +26,7 @@ public void testRepositoryAnalysis() throws Exception { logger.info("creating repository [{}] of type [{}]", repository, repositoryType); registerRepository(repository, repositoryType, true, repositorySettings); - final Request request = new Request(HttpPost.METHOD_NAME, "/_snapshot/" + repository + "/_analyse"); + final Request request = new Request(HttpPost.METHOD_NAME, "/_snapshot/" + repository + "/_analyze"); request.addParameter("blob_count", "10"); request.addParameter("concurrency", "4"); request.addParameter("max_blob_size", "1mb"); From 7b3dabe714762ba914d94055ce8ffe442585bac7 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 11:26:29 +0000 Subject: [PATCH 28/43] Add bugs_detected response field --- .../snapshot-restore/apis/repo-analysis-api.asciidoc | 6 ++++++ .../blobstore/testkit/RepositoryAnalyzeAction.java | 11 ++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc index dd20875dcf031..af9fe3ac8bdbe 100644 --- a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc @@ -274,6 +274,12 @@ The probability of performing rare actions during the test. Equal to the The path in the repository under which all the blobs were written during the test. +`bugs_detected`:: +(list) +A list of correctness issues detected, which will be empty if the API +succeeded. Included to emphasize that a successful response does not guarantee +correct behaviour in future. + `summary`:: (object) A collection of statistics that summarise the results of the test. diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index 98c4cda70abe9..13bf37da084aa 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -874,11 +874,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endObject(); builder.field("repository", repositoryName); - - // TODO add "consistency": "no problems found" - - // TODO wrap the speed test results into an inner object - builder.field("blob_count", blobCount); builder.field("concurrency", concurrency); builder.field("read_node_count", readNodeCount); @@ -888,6 +883,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("seed", seed); builder.field("rare_action_probability", rareActionProbability); builder.field("blob_path", blobPath); + + builder.startArray("bugs_detected"); + // nothing to report here, if we detected a bug then we would have thrown an exception, but we include this to emphasise + // that we are only detecting bugs, not guaranteeing their absence + builder.endArray(); + builder.field("summary", summary); if (blobResponses.size() > 0) { From 9ecfe46b75ac59f6b50eb6fed0594628fae119ad Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 14:55:51 +0000 Subject: [PATCH 29/43] Respects max total size --- .../testkit/RepositoryAnalyzeAction.java | 174 ++++++++++++++++-- .../testkit/RepositoryAnalyzeActionTests.java | 81 ++++++++ 2 files changed, 244 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index 13bf37da084aa..00a73f89f2166 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -57,7 +57,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -68,6 +70,7 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; +import java.util.stream.IntStream; /** * Action which distributes a bunch of {@link BlobAnalyzeAction}s over the nodes in the cluster, with limited concurrency, and collects @@ -185,6 +188,157 @@ private static List getSnapshotNodes(DiscoveryNodes discoveryNode return nodes; } + // Exposed for tests + static List getBlobSizes(Request request) { + + int blobCount = request.getBlobCount(); + long maxTotalBytes = request.getMaxTotalDataSize().getBytes(); + long maxBlobSize = request.getMaxBlobSize().getBytes(); + + if (maxTotalBytes - maxBlobSize < blobCount - 1) { + throw new IllegalArgumentException( + "cannot satisfy max total bytes [" + + maxTotalBytes + + "B/" + + request.getMaxTotalDataSize() + + "]: must write at least one byte per blob and at least one max-sized blob which is [" + + (blobCount + maxBlobSize - 1) + + "B] in total" + ); + } + + final List blobSizes = new ArrayList<>(); + for (long s = 1; 0 < s && s < maxBlobSize; s <<= 1) { + blobSizes.add(s); + } + blobSizes.add(maxBlobSize); + + // Try and form an even spread of blob sizes by accounting for as many evenly spreads repeats as possible up-front. + final long evenSpreadSize = blobSizes.stream().mapToLong(l -> l).sum(); + int evenSpreadCount = 0; + while (blobSizes.size() <= blobCount && blobCount - blobSizes.size() <= maxTotalBytes - evenSpreadSize) { + evenSpreadCount += 1; + maxTotalBytes -= evenSpreadSize; + blobCount -= blobSizes.size(); + } + + if (evenSpreadCount == 0) { + // Not enough bytes for even a single even spread, account for the one max-sized blob anyway. + blobCount -= 1; + maxTotalBytes -= maxBlobSize; + } + + final List perBlobSizes = new BlobCountCalculator(blobCount, maxTotalBytes, blobSizes).calculate(); + + if (evenSpreadCount == 0) { + perBlobSizes.add(maxBlobSize); + } else { + for (final long blobSize : blobSizes) { + for (int i = 0; i < evenSpreadCount; i++) { + perBlobSizes.add(blobSize); + } + } + } + + assert perBlobSizes.size() == request.getBlobCount(); + assert perBlobSizes.stream().mapToLong(l -> l).sum() <= request.getMaxTotalDataSize().getBytes(); + assert perBlobSizes.stream().allMatch(l -> 1 <= l && l <= request.getMaxBlobSize().getBytes()); + assert perBlobSizes.stream().anyMatch(l -> l == request.getMaxBlobSize().getBytes()); + return perBlobSizes; + } + + /** + * Calculates a reasonable set of blob sizes, with the correct number of blobs and a total size that respects the max in the request. + */ + private static class BlobCountCalculator { + + private final int blobCount; + private final long maxTotalBytes; + private final List blobSizes; + private final int sizeCount; + + private final int[] blobsBySize; + private long totalBytes; + private int totalBlobs; + + BlobCountCalculator(int blobCount, long maxTotalBytes, List blobSizes) { + this.blobCount = blobCount; + this.maxTotalBytes = maxTotalBytes; + assert blobCount <= maxTotalBytes; + + this.blobSizes = blobSizes; + sizeCount = blobSizes.size(); + assert sizeCount > 0; + + blobsBySize = new int[sizeCount]; + assert invariant(); + } + + List calculate() { + // add blobs roughly evenly while keeping the total size under maxTotalBytes + addBlobsRoughlyEvenly(sizeCount - 1); + + // repeatedly remove a blob and replace it with some number of smaller blobs, until there are enough blobs + while (totalBlobs < blobCount) { + assert totalBytes <= maxTotalBytes; + + final int minSplitCount = Arrays.stream(blobsBySize).skip(1).allMatch(i -> i <= 1) ? 1 : 2; + final int splitIndex = IntStream.range(1, sizeCount).filter(i -> blobsBySize[i] >= minSplitCount).reduce(-1, (i, j) -> j); + assert splitIndex > 0 : "split at " + splitIndex; + assert blobsBySize[splitIndex] >= minSplitCount; + + final long splitSize = blobSizes.get(splitIndex); + blobsBySize[splitIndex] -= 1; + totalBytes -= splitSize; + totalBlobs -= 1; + + addBlobsRoughlyEvenly(splitIndex - 1); + assert invariant(); + } + + return getPerBlobSizes(); + } + + private List getPerBlobSizes() { + assert invariant(); + + final List perBlobSizes = new ArrayList<>(blobCount); + for (int sizeIndex = 0; sizeIndex < sizeCount; sizeIndex++) { + final long size = blobSizes.get(sizeIndex); + for (int i = 0; i < blobsBySize[sizeIndex]; i++) { + perBlobSizes.add(size); + } + } + + return perBlobSizes; + } + + private void addBlobsRoughlyEvenly(int startingIndex) { + while (totalBlobs < blobCount && totalBytes < maxTotalBytes) { + boolean progress = false; + for (int sizeIndex = startingIndex; 0 <= sizeIndex && totalBlobs < blobCount && totalBytes < maxTotalBytes; sizeIndex--) { + final long size = blobSizes.get(sizeIndex); + if (totalBytes + size <= maxTotalBytes) { + progress = true; + blobsBySize[sizeIndex] += 1; + totalBlobs += 1; + totalBytes += size; + } + } + assert progress; + assert invariant(); + } + } + + private boolean invariant() { + assert IntStream.of(blobsBySize).sum() == totalBlobs : this; + assert IntStream.range(0, sizeCount).mapToLong(i -> blobSizes.get(i) * blobsBySize[i]).sum() == totalBytes : this; + assert totalBlobs <= blobCount : this; + assert totalBytes <= maxTotalBytes : this; + return true; + } + } + public static class AsyncAction { private final TransportService transportService; @@ -304,17 +458,11 @@ public void run() { final Random random = new Random(request.getSeed()); final List nodes = getSnapshotNodes(discoveryNodes); + final List blobSizes = getBlobSizes(request); + Collections.shuffle(blobSizes, random); - final long maxBlobSize = request.getMaxBlobSize().getBytes(); - final List blobSizes = new ArrayList<>(); - for (long s = 1; s < maxBlobSize; s <<= 1) { - blobSizes.add(s); - } - blobSizes.add(maxBlobSize); - - // TODO account for max total blob size for (int i = 0; i < request.getBlobCount(); i++) { - final long targetLength = blobSizes.get(random.nextInt(blobSizes.size())); + final long targetLength = blobSizes.get(i); final boolean smallBlob = targetLength <= Integer.MAX_VALUE; // we only use the non-atomic API for larger blobs final VerifyBlobTask verifyBlobTask = new VerifyBlobTask( nodes.get(random.nextInt(nodes.size())), @@ -325,8 +473,8 @@ public void run() { targetLength, random.nextLong(), nodes, - request.readNodeCount, - request.earlyReadNodeCount, + request.getReadNodeCount(), + request.getEarlyReadNodeCount(), smallBlob && random.nextDouble() < request.getRareActionProbability(), repository.supportURLRepo() && smallBlob && random.nextDouble() < request.getRareActionProbability() ) @@ -613,6 +761,10 @@ public void blobCount(int blobCount) { if (blobCount <= 0) { throw new IllegalArgumentException("blobCount must be >0, but was [" + blobCount + "]"); } + if (blobCount > 100000) { + // Coordination work is O(blobCount) but is supposed to be lightweight, so limit the blob count. + throw new IllegalArgumentException("blobCount must be <= 100000, but was [" + blobCount + "]"); + } this.blobCount = blobCount; } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java new file mode 100644 index 0000000000000..42df4531a492e --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public class RepositoryAnalyzeActionTests extends ESTestCase { + + public void testGetBlobSizesAssertions() { + getBlobSizesOrException(randomLongBetween(1L, Long.MAX_VALUE), randomLongBetween(1L, Long.MAX_VALUE), between(100, 10000)); + } + + public void testGetBlobSizesAssertionsSmall() { + for (long maxBlobSize = 1L; maxBlobSize < 50L; maxBlobSize++) { + for (long maxTotalDataSize = 1L; maxTotalDataSize < 50L; maxTotalDataSize++) { + for (int blobCount = 1; blobCount < 25; blobCount++) { + getBlobSizesOrException(maxBlobSize, maxTotalDataSize, blobCount); + } + } + } + } + + private void getBlobSizesOrException(long maxBlobSize, long maxTotalDataSize, int blobCount) { + try { + getBlobSizes(maxBlobSize, maxTotalDataSize, blobCount); + } catch (IllegalArgumentException e) { + // ok + } + } + + public void testGetBlobSizesExample() { + final int logBlobSize = 10; + final long blobSize = 1 << logBlobSize; + final long totalSize = 2047; + assertThat( + getBlobSizes(blobSize, totalSize, logBlobSize + 1).stream().mapToLong(l -> l).sorted().toArray(), + equalTo(new long[] { 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024 }) // one of each size + ); + } + + public void testGetBlobSizesWithRepeats() { + final int logBlobSize = 10; + final long blobSize = 1 << logBlobSize; + final long totalSize = 6014; // large enough for repeats + assertThat( + getBlobSizes(blobSize, totalSize, 26).stream().mapToLong(l -> l).sorted().toArray(), + equalTo( + new long[] { 1, 1, 2, 2, 4, 4, 8, 8, 16, 16, 32, 32, 64, 64, 128, 128, 128, 256, 256, 256, 512, 512, 512, 1024, 1024, 1024 } + ) + ); + } + + public void testGetBlobSizesTotalTooSmall() { + final int logBlobSize = 10; + final long blobSize = 1 << logBlobSize; + final long totalSize = 2046; // not large enough for one of each size + assertThat( + getBlobSizes(blobSize, totalSize, logBlobSize + 1).stream().mapToLong(l -> l).sorted().toArray(), + equalTo(new long[] { 2, 4, 8, 16, 32, 64, 128, 128, 256, 256, 1024 }) + ); + } + + private static List getBlobSizes(long maxBlobSize, long maxTotalDataSize, int blobCount) { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("repo"); + request.maxBlobSize(new ByteSizeValue(maxBlobSize)); + request.maxTotalDataSize(new ByteSizeValue(maxTotalDataSize)); + request.blobCount(blobCount); + return RepositoryAnalyzeAction.getBlobSizes(request); + } + +} From c4c3cfa31d3cb067cd1eb691e7dbe0c025b94e92 Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 5 Feb 2021 18:08:55 +0000 Subject: [PATCH 30/43] Fix action name --- .../repositories/blobstore/testkit/GetBlobChecksumAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java index 110647fa86b07..08dbd6f688c09 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java @@ -52,7 +52,7 @@ public class GetBlobChecksumAction extends ActionType Date: Fri, 5 Feb 2021 18:48:28 +0000 Subject: [PATCH 31/43] Better tests, corresponding with docs --- .../testkit/RepositoryAnalyzeActionTests.java | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java index 42df4531a492e..80762d6ea1de0 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/test/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeActionTests.java @@ -11,16 +11,20 @@ import org.elasticsearch.test.ESTestCase; import java.util.List; +import java.util.stream.IntStream; +import java.util.stream.LongStream; import static org.hamcrest.Matchers.equalTo; public class RepositoryAnalyzeActionTests extends ESTestCase { public void testGetBlobSizesAssertions() { + // Just checking that no assertions are thrown getBlobSizesOrException(randomLongBetween(1L, Long.MAX_VALUE), randomLongBetween(1L, Long.MAX_VALUE), between(100, 10000)); } public void testGetBlobSizesAssertionsSmall() { + // Just checking that no assertions are thrown for an exhaustive set of small examples for (long maxBlobSize = 1L; maxBlobSize < 50L; maxBlobSize++) { for (long maxTotalDataSize = 1L; maxTotalDataSize < 50L; maxTotalDataSize++) { for (int blobCount = 1; blobCount < 25; blobCount++) { @@ -30,15 +34,7 @@ public void testGetBlobSizesAssertionsSmall() { } } - private void getBlobSizesOrException(long maxBlobSize, long maxTotalDataSize, int blobCount) { - try { - getBlobSizes(maxBlobSize, maxTotalDataSize, blobCount); - } catch (IllegalArgumentException e) { - // ok - } - } - - public void testGetBlobSizesExample() { + public void testGetBlobSizesWithoutRepeats() { final int logBlobSize = 10; final long blobSize = 1 << logBlobSize; final long totalSize = 2047; @@ -70,6 +66,40 @@ public void testGetBlobSizesTotalTooSmall() { ); } + public void testGetBlobSizesDefaultExample() { + // The defaults are 10MB max blob size, 1GB max total size, 100 blobs, which gives 4 blobs of each size (either powers of 2 or 10MB) + assertThat( + RepositoryAnalyzeAction.getBlobSizes(new RepositoryAnalyzeAction.Request("repo")).stream().mapToLong(l -> l).sorted().toArray(), + equalTo( + LongStream.range(0, 25) + .flatMap(power -> IntStream.range(0, 4).mapToLong(i -> power == 24 ? ByteSizeValue.ofMb(10).getBytes() : 1L << power)) + .toArray() + ) + ); + } + + public void testGetBlobSizesRealisticExample() { + // The docs suggest a realistic example needs 2GB max blob size, 1TB max total size, 2000 blobs, which gives 62 or 63 blobs of each + // size (powers of 2 up to 2GB) + assertThat( + getBlobSizes(ByteSizeValue.ofGb(2).getBytes(), ByteSizeValue.ofTb(1).getBytes(), 2000).stream() + .mapToLong(l -> l) + .sorted() + .toArray(), + equalTo( + LongStream.range(0, 32).flatMap(power -> IntStream.range(0, power < 16 ? 62 : 63).mapToLong(i -> 1L << power)).toArray() + ) + ); + } + + private static void getBlobSizesOrException(long maxBlobSize, long maxTotalDataSize, int blobCount) { + try { + getBlobSizes(maxBlobSize, maxTotalDataSize, blobCount); + } catch (IllegalArgumentException e) { + // ok + } + } + private static List getBlobSizes(long maxBlobSize, long maxTotalDataSize, int blobCount) { final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("repo"); request.maxBlobSize(new ByteSizeValue(maxBlobSize)); From 59ff6ec54d27e2c5437fe3dae732adb8b8026324 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 8 Feb 2021 17:02:14 +0000 Subject: [PATCH 32/43] No need for retries when listing --- .../testkit/RepositoryAnalyzeAction.java | 148 ++++++------------ 1 file changed, 50 insertions(+), 98 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index 00a73f89f2166..ecc447833b2a9 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -20,9 +20,7 @@ import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.HandledTransportAction; -import org.elasticsearch.action.support.ThreadedActionListener; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -37,6 +35,7 @@ import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.CountDown; import org.elasticsearch.common.xcontent.StatusToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.repositories.RepositoriesService; @@ -58,12 +57,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.OptionalLong; import java.util.Queue; import java.util.Random; import java.util.Set; @@ -357,16 +354,11 @@ public static class AsyncAction { private final AtomicReference failure = new AtomicReference<>(); private final Semaphore innerFailures = new Semaphore(5); // limit the number of suppressed failures private final int workerCount; - private final GroupedActionListener workersListener; + private final CountDown workerCountdown; private final Set expectedBlobs = ConcurrentCollections.newConcurrentSet(); private final List responses; private final RepositoryPerformanceSummary.Builder summary = new RepositoryPerformanceSummary.Builder(); - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - private OptionalLong listingStartTimeNanos = OptionalLong.empty(); - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - private OptionalLong deleteStartTimeNanos = OptionalLong.empty(); - public AsyncAction( TransportService transportService, BlobStoreRepository repository, @@ -386,21 +378,7 @@ public AsyncAction( this.listener = listener; this.workerCount = request.getConcurrency(); - this.workersListener = new GroupedActionListener<>( - new ThreadedActionListener<>(logger, transportService.getThreadPool(), ThreadPool.Names.SNAPSHOT, new ActionListener<>() { - @Override - public void onResponse(Collection voids) { - onWorkerCompletion(); - } - - @Override - public void onFailure(Exception e) { - assert false : e; // workers should not fail - onWorkerCompletion(); - } - }, false), - workerCount - ); + this.workerCountdown = new CountDown(workerCount); responses = new ArrayList<>(request.blobCount); } @@ -451,7 +429,9 @@ private boolean isRunning() { } public void run() { - assert queue.isEmpty() && failure.get() == null : "must only run action once"; + assert queue.isEmpty() : "must only run action once"; + assert failure.get() == null : "must only run action once"; + assert workerCountdown.isCountedDown() == false : "must only run action once"; logger.info("running analysis of repository [{}] using path [{}]", request.getRepositoryName(), blobPath); @@ -490,7 +470,7 @@ public void run() { private void processNextTask() { final VerifyBlobTask thisTask = queue.poll(); if (isRunning() == false || thisTask == null) { - workersListener.onResponse(null); + onWorkerCompletion(); } else { logger.trace("processing [{}]", thisTask); // NB although all this is on the SAME thread, the per-blob verification runs on a SNAPSHOT thread so we don't have to worry @@ -522,7 +502,7 @@ public void handleResponse(BlobAnalyzeAction.Response response) { public void handleException(TransportException exp) { logger.debug(new ParameterizedMessage("failed [{}]", thisTask), exp); fail(exp); - workersListener.onResponse(null); + onWorkerCompletion(); } @Override @@ -540,108 +520,80 @@ private BlobContainer getBlobContainer() { } private void onWorkerCompletion() { - tryListAndCleanup(0); + if (workerCountdown.countDown()) { + transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT).execute(() -> { + final long listingStartTimeNanos = System.nanoTime(); + ensureConsistentListing(); + final long deleteStartTimeNanos = System.nanoTime(); + deleteContainer(); + sendResponse(listingStartTimeNanos, deleteStartTimeNanos); + }); + } } - private void tryListAndCleanup(final int retryCount) { - // TODO no need for retries any more now that S3 has consistent listings + private void ensureConsistentListing() { if (timeoutTimeMillis < currentTimeMillisSupplier.getAsLong() || task.isCancelled()) { logger.warn( - "analysis of repository [{}] failed in cleanup phase after [{}] attempts, attempting best-effort cleanup " + "analysis of repository [{}] failed before cleanup phase, attempting best-effort cleanup " + "but you may need to manually remove [{}]", request.getRepositoryName(), - retryCount, blobPath ); isRunning(); // set failure if not already set - deleteContainerAndSendResponse(); } else { - boolean retry = true; + logger.trace( + "all tasks completed, checking expected blobs exist in [{}:{}] before cleanup", + request.repositoryName, + blobPath + ); try { - if (listingStartTimeNanos.isEmpty()) { - assert retryCount == 0 : retryCount; - logger.trace( - "all tasks completed, checking expected blobs exist in [{}:{}] before cleanup", - request.repositoryName, - blobPath - ); - listingStartTimeNanos = OptionalLong.of(System.nanoTime()); - } else { - logger.trace( - "retrying check that expected blobs exist in [{}:{}] before cleanup, retry count = {}", - request.repositoryName, - blobPath, - retryCount - ); - } final BlobContainer blobContainer = getBlobContainer(); - final Map blobsMap = blobContainer.listBlobs(); - final Set missingBlobs = new HashSet<>(expectedBlobs); + final Map blobsMap = blobContainer.listBlobs(); missingBlobs.removeAll(blobsMap.keySet()); + if (missingBlobs.isEmpty()) { - logger.trace( - "all expected blobs found after {} failed attempts, cleaning up [{}:{}]", - retryCount, - request.getRepositoryName(), - blobPath - ); - retry = false; - deleteContainerAndSendResponse(); + logger.trace("all expected blobs found, cleaning up [{}:{}]", request.getRepositoryName(), blobPath); } else { - // TODO do we need this retry mechanism any more? S3 used not to have consistent listings but it might be ok now. - logger.debug( - "expected blobs [{}] missing in [{}:{}], trying again; retry count = {}", + final RepositoryVerificationException repositoryVerificationException = new RepositoryVerificationException( request.repositoryName, - blobPath, - missingBlobs, - retryCount + "expected blobs " + missingBlobs + " missing in [" + request.repositoryName + ":" + blobPath + "]" ); + logger.debug("failing due to missing blobs", repositoryVerificationException); + fail(repositoryVerificationException); } } catch (Exception e) { - assert retry; - logger.debug( - new ParameterizedMessage( - "failure during cleanup of [{}:{}], will retry; retry count = {}", - request.getRepositoryName(), - blobPath, - retryCount - ), - e - ); + logger.debug(new ParameterizedMessage("failure during cleanup of [{}:{}]", request.getRepositoryName(), blobPath), e); fail(e); - } finally { - if (retry) { - // Either there were missing blobs, or else listing the blobs failed. - transportService.getThreadPool() - .scheduleUnlessShuttingDown( - TimeValue.timeValueSeconds(10), - ThreadPool.Names.SNAPSHOT, - () -> tryListAndCleanup(retryCount + 1) - ); - } } } } - private void deleteContainerAndSendResponse() { + private void deleteContainer() { try { - assert deleteStartTimeNanos.isEmpty() : deleteStartTimeNanos; - deleteStartTimeNanos = OptionalLong.of(System.nanoTime()); getBlobContainer().delete(); - // TODO can we check that the delete actually succeeded here? + final BlobContainer blobContainer = getBlobContainer(); + blobContainer.delete(); + if (failure.get() != null) { + return; + } + final Map blobsMap = blobContainer.listBlobs(); + if (blobsMap.isEmpty() == false) { + final RepositoryVerificationException repositoryVerificationException = new RepositoryVerificationException( + request.repositoryName, + "failed to clean up blobs " + blobsMap.keySet() + ); + logger.debug("failing due to leftover blobs", repositoryVerificationException); + fail(repositoryVerificationException); + } } catch (Exception e) { fail(e); - } finally { - sendResponse(); } } - private void sendResponse() { + private void sendResponse(final long listingStartTimeNanos, final long deleteStartTimeNanos) { final Exception exception = failure.get(); if (exception == null) { - assert listingStartTimeNanos.isPresent(); - assert deleteStartTimeNanos.isPresent(); final long completionTimeNanos = System.nanoTime(); logger.trace("[{}] completed successfully", request.getDescription()); @@ -662,8 +614,8 @@ private void sendResponse() { blobPath, summary.build(), responses, - deleteStartTimeNanos.getAsLong() - listingStartTimeNanos.getAsLong(), - completionTimeNanos - deleteStartTimeNanos.getAsLong() + deleteStartTimeNanos - listingStartTimeNanos, + completionTimeNanos - deleteStartTimeNanos ) ); } else { From 28b488b00da33c8816969075b518eac2d7ed9c1a Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 15 Feb 2021 13:50:26 +0000 Subject: [PATCH 33/43] Add integ tests for failure and success cases --- .../testkit/RepositoryAnalysisFailureIT.java | 470 ++++++++++++++++++ ....java => RepositoryAnalysisSuccessIT.java} | 111 +++-- .../blobstore/testkit/BlobAnalyzeAction.java | 15 +- .../testkit/RepositoryAnalyzeAction.java | 5 +- 4 files changed, 561 insertions(+), 40 deletions(-) create mode 100644 x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisFailureIT.java rename x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/{RepositoryAnalysisIT.java => RepositoryAnalysisSuccessIT.java} (74%) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisFailureIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisFailureIT.java new file mode 100644 index 0000000000000..00e14dcabd288 --- /dev/null +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisFailureIT.java @@ -0,0 +1,470 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.repositories.blobstore.testkit; + +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.cluster.metadata.RepositoryMetadata; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobMetadata; +import org.elasticsearch.common.blobstore.BlobPath; +import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.DeleteResult; +import org.elasticsearch.common.blobstore.support.PlainBlobMetadata; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.CountDown; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.indices.recovery.RecoverySettings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.RepositoryPlugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.repositories.Repository; +import org.elasticsearch.repositories.RepositoryMissingException; +import org.elasticsearch.repositories.RepositoryVerificationException; +import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase; +import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; +import org.junit.Before; + +import java.io.ByteArrayInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileAlreadyExistsException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.stream.Collectors; + +import static org.hamcrest.Matchers.anEmptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class RepositoryAnalysisFailureIT extends AbstractSnapshotIntegTestCase { + + private DisruptableBlobStore blobStore; + + @Before + public void suppressConsistencyChecks() { + disableRepoConsistencyCheck("repository is not used for snapshots"); + } + + @Override + protected Collection> nodePlugins() { + return List.of(TestPlugin.class, LocalStateCompositeXPackPlugin.class, SnapshotRepositoryTestKit.class); + } + + @Before + public void createBlobStore() { + createRepositoryNoVerify("test-repo", TestPlugin.DISRUPTABLE_REPO_TYPE); + + blobStore = new DisruptableBlobStore(); + for (final RepositoriesService repositoriesService : internalCluster().getInstances(RepositoriesService.class)) { + try { + ((DisruptableRepository) repositoriesService.repository("test-repo")).setBlobStore(blobStore); + } catch (RepositoryMissingException e) { + // it's only present on voting masters and data nodes + } + } + } + + public void testSuccess() { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); + request.blobCount(1); + request.maxBlobSize(new ByteSizeValue(10L)); + + final RepositoryAnalyzeAction.Response response = analyseRepository(request); + assertThat(response.status(), equalTo(RestStatus.OK)); + } + + public void testFailsOnReadError() { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); + request.maxBlobSize(new ByteSizeValue(10L)); + + final CountDown countDown = new CountDown(between(1, request.getBlobCount())); + blobStore.setDisruption(new Disruption() { + @Override + public byte[] onRead(byte[] actualContents, long position, long length) throws IOException { + if (countDown.countDown()) { + throw new IOException("simulated"); + } + return actualContents; + } + }); + + final Exception exception = expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request)); + final IOException ioException = (IOException) ExceptionsHelper.unwrap(exception, IOException.class); + assert ioException != null : exception; + assertThat(ioException.getMessage(), equalTo("simulated")); + } + + public void testFailsOnNotFoundAfterWrite() { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); + request.maxBlobSize(new ByteSizeValue(10L)); + request.earlyReadNodeCount(0); // not found on early read is ok + + final CountDown countDown = new CountDown(between(1, request.getBlobCount())); + + blobStore.setDisruption(new Disruption() { + @Override + public byte[] onRead(byte[] actualContents, long position, long length) { + if (countDown.countDown()) { + return null; + } + return actualContents; + } + }); + + expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request)); + } + + public void testFailsOnChecksumMismatch() { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); + request.maxBlobSize(new ByteSizeValue(10L)); + + final CountDown countDown = new CountDown(between(1, request.getBlobCount())); + + blobStore.setDisruption(new Disruption() { + @Override + public byte[] onRead(byte[] actualContents, long position, long length) { + final byte[] disruptedContents = actualContents == null ? null : Arrays.copyOf(actualContents, actualContents.length); + if (actualContents != null && countDown.countDown()) { + // CRC32 should always detect a single bit flip + disruptedContents[Math.toIntExact(position + randomLongBetween(0, length - 1))] ^= 1 << between(0, 7); + } + return disruptedContents; + } + }); + + expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request)); + } + + public void testFailsOnWriteException() { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); + request.maxBlobSize(new ByteSizeValue(10L)); + + final CountDown countDown = new CountDown(between(1, request.getBlobCount())); + + blobStore.setDisruption(new Disruption() { + + @Override + public void onWrite() throws IOException { + if (countDown.countDown()) { + throw new IOException("simulated"); + } + } + + }); + + final Exception exception = expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request)); + final IOException ioException = (IOException) ExceptionsHelper.unwrap(exception, IOException.class); + assert ioException != null : exception; + assertThat(ioException.getMessage(), equalTo("simulated")); + } + + public void testFailsOnIncompleteListing() { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); + request.maxBlobSize(new ByteSizeValue(10L)); + + blobStore.setDisruption(new Disruption() { + + @Override + public Map onList(Map actualListing) { + final HashMap listing = new HashMap<>(actualListing); + listing.keySet().iterator().remove(); + return listing; + } + + }); + + expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request)); + } + + public void testFailsOnListingException() { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); + request.maxBlobSize(new ByteSizeValue(10L)); + + final CountDown countDown = new CountDown(1); + blobStore.setDisruption(new Disruption() { + + @Override + public Map onList(Map actualListing) throws IOException { + if (countDown.countDown()) { + throw new IOException("simulated"); + } + return actualListing; + } + }); + + expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request)); + } + + public void testFailsOnDeleteException() { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); + request.maxBlobSize(new ByteSizeValue(10L)); + + blobStore.setDisruption(new Disruption() { + @Override + public void onDelete() throws IOException { + throw new IOException("simulated"); + } + }); + + expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request)); + } + + public void testFailsOnIncompleteDelete() { + final RepositoryAnalyzeAction.Request request = new RepositoryAnalyzeAction.Request("test-repo"); + request.maxBlobSize(new ByteSizeValue(10L)); + + blobStore.setDisruption(new Disruption() { + + volatile boolean isDeleted; + + @Override + public void onDelete() { + isDeleted = true; + } + + @Override + public Map onList(Map actualListing) { + if (isDeleted) { + assertThat(actualListing, anEmptyMap()); + return Collections.singletonMap("leftover", new PlainBlobMetadata("leftover", 1)); + } else { + return actualListing; + } + } + }); + + expectThrows(RepositoryVerificationException.class, () -> analyseRepository(request)); + } + + private RepositoryAnalyzeAction.Response analyseRepository(RepositoryAnalyzeAction.Request request) { + return client().execute(RepositoryAnalyzeAction.INSTANCE, request).actionGet(30L, TimeUnit.SECONDS); + } + + public static class TestPlugin extends Plugin implements RepositoryPlugin { + + static final String DISRUPTABLE_REPO_TYPE = "disruptable"; + + @Override + public Map getRepositories( + Environment env, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + BigArrays bigArrays, + RecoverySettings recoverySettings + ) { + return Map.of( + DISRUPTABLE_REPO_TYPE, + metadata -> new DisruptableRepository( + metadata, + namedXContentRegistry, + clusterService, + bigArrays, + recoverySettings, + new BlobPath() + ) + ); + } + } + + static class DisruptableRepository extends BlobStoreRepository { + + private final AtomicReference blobStoreRef = new AtomicReference<>(); + + DisruptableRepository( + RepositoryMetadata metadata, + NamedXContentRegistry namedXContentRegistry, + ClusterService clusterService, + BigArrays bigArrays, + RecoverySettings recoverySettings, + BlobPath basePath + ) { + super(metadata, namedXContentRegistry, clusterService, bigArrays, recoverySettings, basePath); + } + + void setBlobStore(BlobStore blobStore) { + assertTrue(blobStoreRef.compareAndSet(null, blobStore)); + } + + @Override + protected BlobStore createBlobStore() { + final BlobStore blobStore = blobStoreRef.get(); + assertNotNull(blobStore); + return blobStore; + } + } + + static class DisruptableBlobStore implements BlobStore { + + @Nullable // if deleted + private DisruptableBlobContainer blobContainer; + + private Disruption disruption = Disruption.NONE; + + @Override + public BlobContainer blobContainer(BlobPath path) { + synchronized (this) { + if (blobContainer == null) { + blobContainer = new DisruptableBlobContainer(path, this::deleteContainer, disruption); + } + return blobContainer; + } + } + + private void deleteContainer(DisruptableBlobContainer container) { + blobContainer = null; + } + + @Override + public void close() {} + + public void setDisruption(Disruption disruption) { + assertThat("cannot change disruption while blob container exists", blobContainer, nullValue()); + this.disruption = disruption; + } + } + + interface Disruption { + + Disruption NONE = new Disruption() { + }; + + default byte[] onRead(byte[] actualContents, long position, long length) throws IOException { + return actualContents; + } + + default void onWrite() throws IOException {} + + default Map onList(Map actualListing) throws IOException { + return actualListing; + } + + default void onDelete() throws IOException {} + } + + static class DisruptableBlobContainer implements BlobContainer { + + private final BlobPath path; + private final Consumer deleteContainer; + private final Disruption disruption; + private final Map blobs = ConcurrentCollections.newConcurrentMap(); + + DisruptableBlobContainer(BlobPath path, Consumer deleteContainer, Disruption disruption) { + this.path = path; + this.deleteContainer = deleteContainer; + this.disruption = disruption; + } + + @Override + public BlobPath path() { + return path; + } + + @Override + public boolean blobExists(String blobName) { + return blobs.containsKey(blobName); + } + + @Override + public InputStream readBlob(String blobName) throws IOException { + final byte[] actualContents = blobs.get(blobName); + final byte[] disruptedContents = disruption.onRead(actualContents, 0L, actualContents == null ? 0L : actualContents.length); + if (disruptedContents == null) { + throw new FileNotFoundException(blobName + " not found"); + } + return new ByteArrayInputStream(disruptedContents); + } + + @Override + public InputStream readBlob(String blobName, long position, long length) throws IOException { + final byte[] actualContents = blobs.get(blobName); + final byte[] disruptedContents = disruption.onRead(actualContents, position, length); + if (disruptedContents == null) { + throw new FileNotFoundException(blobName + " not found"); + } + final int truncatedLength = Math.toIntExact(Math.min(length, disruptedContents.length - position)); + return new ByteArrayInputStream(disruptedContents, Math.toIntExact(position), truncatedLength); + } + + @Override + public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { + writeBlobAtomic(blobName, inputStream, failIfAlreadyExists); + } + + @Override + public void writeBlob(String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + writeBlob(blobName, bytes.streamInput(), bytes.length(), failIfAlreadyExists); + } + + @Override + public void writeBlobAtomic(String blobName, BytesReference bytes, boolean failIfAlreadyExists) throws IOException { + writeBlobAtomic(blobName, bytes.streamInput(), failIfAlreadyExists); + } + + private void writeBlobAtomic(String blobName, InputStream inputStream, boolean failIfAlreadyExists) throws IOException { + if (failIfAlreadyExists && blobs.get(blobName) != null) { + throw new FileAlreadyExistsException(blobName); + } + + final byte[] contents = inputStream.readAllBytes(); + disruption.onWrite(); + blobs.put(blobName, contents); + } + + @Override + public DeleteResult delete() throws IOException { + disruption.onDelete(); + deleteContainer.accept(this); + final DeleteResult deleteResult = new DeleteResult(blobs.size(), blobs.values().stream().mapToLong(b -> b.length).sum()); + blobs.clear(); + return deleteResult; + } + + @Override + public void deleteBlobsIgnoringIfNotExists(List blobNames) { + blobs.keySet().removeAll(blobNames); + } + + @Override + public Map listBlobs() throws IOException { + return disruption.onList( + blobs.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new PlainBlobMetadata(e.getKey(), e.getValue().length))) + ); + } + + @Override + public Map children() { + return Map.of(); + } + + @Override + public Map listBlobsByPrefix(String blobNamePrefix) throws IOException { + final Map blobMetadataByName = listBlobs(); + blobMetadataByName.keySet().removeIf(s -> s.startsWith(blobNamePrefix) == false); + return blobMetadataByName; + } + } + +} diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisSuccessIT.java similarity index 74% rename from x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java rename to x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisSuccessIT.java index 7f20f45222fbe..16e833292d161 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisIT.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/internalClusterTest/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalysisSuccessIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobMetadata; import org.elasticsearch.common.blobstore.BlobPath; @@ -42,18 +43,18 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; -public class RepositoryAnalysisIT extends AbstractSnapshotIntegTestCase { - - // These tests are incomplete and do not currently verify that the speed test picks up on any particular repository-side failures - // TODO complete these tests +public class RepositoryAnalysisSuccessIT extends AbstractSnapshotIntegTestCase { @Before public void suppressConsistencyChecks() { @@ -65,16 +66,16 @@ protected Collection> nodePlugins() { return List.of(TestPlugin.class, LocalStateCompositeXPackPlugin.class, SnapshotRepositoryTestKit.class); } - public void testFoo() { + public void testRepositoryAnalysis() { - createRepositoryNoVerify("test-repo", TestPlugin.DISRUPTABLE_REPO_TYPE); + createRepositoryNoVerify("test-repo", TestPlugin.ASSERTING_REPO_TYPE); - final DisruptableBlobStore blobStore = new DisruptableBlobStore(); + final AssertingBlobStore blobStore = new AssertingBlobStore(); for (final RepositoriesService repositoriesService : internalCluster().getInstances(RepositoriesService.class)) { try { - ((DisruptableRepository) repositoriesService.repository("test-repo")).setBlobStore(blobStore); + ((AssertingRepository) repositoriesService.repository("test-repo")).setBlobStore(blobStore); } catch (RepositoryMissingException e) { - // it's only present on voting masters and data nodes + // nbd, it's only present on voting masters and data nodes } } @@ -96,16 +97,23 @@ public void testFoo() { blobStore.ensureMaxBlobSize(request.getMaxBlobSize().getBytes()); } + if (usually()) { + request.maxTotalDataSize(new ByteSizeValue(between(1, 1 << 20))); + blobStore.ensureMaxTotalBlobSize(request.getMaxTotalDataSize().getBytes()); + } + request.timeout(TimeValue.timeValueSeconds(5)); - final RepositoryAnalyzeAction.Response response = client().execute(RepositoryAnalyzeAction.INSTANCE, request).actionGet(); + final RepositoryAnalyzeAction.Response response = client().execute(RepositoryAnalyzeAction.INSTANCE, request) + .actionGet(30L, TimeUnit.SECONDS); assertThat(response.status(), equalTo(RestStatus.OK)); + assertThat(blobStore.currentPath, nullValue()); } public static class TestPlugin extends Plugin implements RepositoryPlugin { - static final String DISRUPTABLE_REPO_TYPE = "disruptable"; + static final String ASSERTING_REPO_TYPE = "asserting"; @Override public Map getRepositories( @@ -116,8 +124,8 @@ public Map getRepositories( RecoverySettings recoverySettings ) { return Map.of( - DISRUPTABLE_REPO_TYPE, - metadata -> new DisruptableRepository( + ASSERTING_REPO_TYPE, + metadata -> new AssertingRepository( metadata, namedXContentRegistry, clusterService, @@ -129,11 +137,11 @@ public Map getRepositories( } } - static class DisruptableRepository extends BlobStoreRepository { + static class AssertingRepository extends BlobStoreRepository { private final AtomicReference blobStoreRef = new AtomicReference<>(); - DisruptableRepository( + AssertingRepository( RepositoryMetadata metadata, NamedXContentRegistry namedXContentRegistry, ClusterService clusterService, @@ -154,38 +162,55 @@ protected BlobStore createBlobStore() { assertNotNull(blobStore); return blobStore; } - - @Override - public String startVerification() { - return null; // suppress verify-on-register since we need to wire up all the nodes' repositories first - } } - static class DisruptableBlobStore implements BlobStore { + static class AssertingBlobStore implements BlobStore { + + @Nullable // if no current blob container + private String currentPath; + + @Nullable // if no current blob container + private AssertingBlobContainer currentBlobContainer; - private final Map blobContainers = ConcurrentCollections.newConcurrentMap(); private Semaphore writeSemaphore = new Semaphore(new RepositoryAnalyzeAction.Request("dummy").getConcurrency()); private int maxBlobCount = new RepositoryAnalyzeAction.Request("dummy").getBlobCount(); private long maxBlobSize = new RepositoryAnalyzeAction.Request("dummy").getMaxBlobSize().getBytes(); + private long maxTotalBlobSize = new RepositoryAnalyzeAction.Request("dummy").getMaxTotalDataSize().getBytes(); @Override public BlobContainer blobContainer(BlobPath path) { assertThat(path.buildAsString(), startsWith("temp-analysis-")); - return blobContainers.computeIfAbsent( - path.buildAsString(), - ignored -> new DisruptableBlobContainer(path, this::deleteContainer, writeSemaphore, maxBlobCount, maxBlobSize) - ); + + synchronized (this) { + if (currentPath == null) { + currentPath = path.buildAsString(); + currentBlobContainer = new AssertingBlobContainer( + path, + this::deleteContainer, + writeSemaphore, + maxBlobCount, + maxBlobSize, + maxTotalBlobSize + ); + } + assertThat(path.buildAsString(), equalTo(currentPath)); + return currentBlobContainer; + } } - private void deleteContainer(DisruptableBlobContainer container) { - assertNotNull("container " + container.path() + " not found", blobContainers.remove(container.path().buildAsString())); + private void deleteContainer(AssertingBlobContainer container) { + synchronized (this) { + assertThat(currentPath, equalTo(container.path.buildAsString())); + currentPath = null; + currentBlobContainer = null; + } } @Override public void close() {} public void ensureMaxWriteConcurrency(int concurrency) { - writeSemaphore = new Semaphore(concurrency); + this.writeSemaphore = new Semaphore(concurrency); } public void ensureMaxBlobCount(int maxBlobCount) { @@ -195,31 +220,39 @@ public void ensureMaxBlobCount(int maxBlobCount) { public void ensureMaxBlobSize(long maxBlobSize) { this.maxBlobSize = maxBlobSize; } + + public void ensureMaxTotalBlobSize(long maxTotalBlobSize) { + this.maxTotalBlobSize = maxTotalBlobSize; + } } - static class DisruptableBlobContainer implements BlobContainer { + static class AssertingBlobContainer implements BlobContainer { private static final byte[] EMPTY = new byte[0]; private final BlobPath path; - private final Consumer deleteContainer; + private final Consumer deleteContainer; private final Semaphore writeSemaphore; private final int maxBlobCount; private final long maxBlobSize; + private final long maxTotalBlobSize; private final Map blobs = ConcurrentCollections.newConcurrentMap(); + private final AtomicLong totalBytesWritten = new AtomicLong(); - DisruptableBlobContainer( + AssertingBlobContainer( BlobPath path, - Consumer deleteContainer, + Consumer deleteContainer, Semaphore writeSemaphore, int maxBlobCount, - long maxBlobSize + long maxBlobSize, + long maxTotalBlobSize ) { this.path = path; this.deleteContainer = deleteContainer; this.writeSemaphore = writeSemaphore; this.maxBlobCount = maxBlobCount; this.maxBlobSize = maxBlobSize; + this.maxTotalBlobSize = maxTotalBlobSize; } @Override @@ -273,9 +306,11 @@ public void writeBlobAtomic(String blobName, BytesReference bytes, boolean failI private void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException { + final byte[] existingBlob = blobs.get(blobName); if (failIfAlreadyExists) { - assertNull("blob [" + blobName + "] must not exist", blobs.get(blobName)); + assertNull("blob [" + blobName + "] must not exist", existingBlob); } + final int existingSize = existingBlob == null ? 0 : existingBlob.length; assertThat(blobSize, lessThanOrEqualTo(maxBlobSize)); @@ -285,6 +320,8 @@ private void writeBlobAtomic(String blobName, InputStream inputStream, long blob assertThat((long) contents.length, equalTo(blobSize)); blobs.put(blobName, contents); assertThat(blobs.size(), lessThanOrEqualTo(maxBlobCount)); + final long currentTotal = totalBytesWritten.addAndGet(blobSize - existingSize); + assertThat(currentTotal, lessThanOrEqualTo(maxTotalBlobSize)); } finally { writeSemaphore.release(); } @@ -293,7 +330,9 @@ private void writeBlobAtomic(String blobName, InputStream inputStream, long blob @Override public DeleteResult delete() { deleteContainer.accept(this); - return new DeleteResult(blobs.size(), blobs.values().stream().mapToLong(b -> b.length).sum()); + final DeleteResult deleteResult = new DeleteResult(blobs.size(), blobs.values().stream().mapToLong(b -> b.length).sum()); + blobs.clear(); + return deleteResult; } @Override diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java index bd6b5233055b5..ba700a5c9134f 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java @@ -365,7 +365,20 @@ private void readOnNodes(List nodes, boolean beforeWriteComplete) new ActionListenerResponseHandler<>(new ActionListener<>() { @Override public void onResponse(GetBlobChecksumAction.Response response) { - readNodesListener.onResponse(makeNodeResponse(node, beforeWriteComplete, response)); + if (beforeWriteComplete == false && response.isNotFound()) { + readNodesListener.onFailure( + new RepositoryVerificationException( + request.getRepositoryName(), + "[" + + blobChecksumRequest + + "] (after write complete) failed on node [" + + node + + "]: blob not found" + ) + ); + } else { + readNodesListener.onResponse(makeNodeResponse(node, beforeWriteComplete, response)); + } } @Override diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index ecc447833b2a9..2ff23cf211cae 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -571,7 +571,6 @@ private void ensureConsistentListing() { private void deleteContainer() { try { - getBlobContainer().delete(); final BlobContainer blobContainer = getBlobContainer(); blobContainer.delete(); if (failure.get() != null) { @@ -805,8 +804,8 @@ public int getReadNodeCount() { } public void earlyReadNodeCount(int earlyReadNodeCount) { - if (earlyReadNodeCount <= 0) { - throw new IllegalArgumentException("earlyReadNodeCount must be >0, but was [" + earlyReadNodeCount + "]"); + if (earlyReadNodeCount < 0) { + throw new IllegalArgumentException("earlyReadNodeCount must be >=0, but was [" + earlyReadNodeCount + "]"); } this.earlyReadNodeCount = earlyReadNodeCount; } From b3833258dfb2f60e5c281c68b693e16279ed1819 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 15 Feb 2021 15:14:10 +0000 Subject: [PATCH 34/43] Unrelated change --- .../CompatibleVersionHelperTests.java.new | 389 ++++++++++++++++++ 1 file changed, 389 insertions(+) create mode 100644 server/src/test/java/org/elasticsearch/rest/CompatibleVersionHelperTests.java.new diff --git a/server/src/test/java/org/elasticsearch/rest/CompatibleVersionHelperTests.java.new b/server/src/test/java/org/elasticsearch/rest/CompatibleVersionHelperTests.java.new new file mode 100644 index 0000000000000..f96451e74d37f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/rest/CompatibleVersionHelperTests.java.new @@ -0,0 +1,389 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.rest; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.Version; +import org.elasticsearch.common.xcontent.ParsedMediaType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.hamcrest.ElasticsearchMatchers; +import org.hamcrest.Matcher; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; + +public class CompatibleVersionHelperTests extends ESTestCase { + int CURRENT_VERSION = Version.CURRENT.major; + int PREVIOUS_VERSION = Version.CURRENT.major - 1; + int OBSOLETE_VERSION = Version.CURRENT.major - 2; + + public void testAcceptAndContentTypeCombinations() { + assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()), isCompatible()); + + assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()), isCompatible()); + + ElasticsearchStatusException e = expectThrows( + ElasticsearchStatusException.class, + () -> requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent()) + ); + assertThat( + e.getMessage(), + equalTo( + "A compatible version is required on both Content-Type and Accept headers " + + "if either one has requested a compatible version and the compatible versions must match. " + + "Accept=" + + acceptHeader(PREVIOUS_VERSION) + + ", Content-Type=" + + contentTypeHeader(CURRENT_VERSION) + ) + ); + + // no body - content-type is ignored + assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent()), isCompatible()); + // no body - content-type is ignored + assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()), not(isCompatible())); + + e = expectThrows( + ElasticsearchStatusException.class, + () -> requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()) + ); + assertThat( + e.getMessage(), + equalTo( + "A compatible version is required on both Content-Type and Accept headers " + + "if either one has requested a compatible version and the compatible versions must match. " + + "Accept=" + + acceptHeader(CURRENT_VERSION) + + ", Content-Type=" + + contentTypeHeader(PREVIOUS_VERSION) + ) + ); + + assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyPresent()), not(isCompatible())); + + assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(CURRENT_VERSION), bodyNotPresent()), not(isCompatible())); + + // tests when body present and one of the headers missing - versioning is required on both when body is present + e = expectThrows( + ElasticsearchStatusException.class, + () -> requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyPresent()) + ); + assertThat( + e.getMessage(), + equalTo( + "A compatible version is required on both Content-Type and Accept headers " + + "if either one has requested a compatible version and the compatible versions must match. " + + "Accept=" + + acceptHeader(PREVIOUS_VERSION) + + ", Content-Type=" + + contentTypeHeader(null) + ) + ); + + e = expectThrows( + ElasticsearchStatusException.class, + () -> requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyPresent()) + ); + assertThat( + e.getMessage(), + equalTo( + "A compatible version is required on both Content-Type and Accept headers " + + "if either one has requested a compatible version. " + + "Accept=" + + acceptHeader(CURRENT_VERSION) + + ", Content-Type=" + + contentTypeHeader(null) + ) + ); + + e = expectThrows( + ElasticsearchStatusException.class, + () -> requestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyPresent()) + ); + assertThat( + e.getMessage(), + equalTo( + "A compatible version is required on both Content-Type and Accept headers " + + "if either one has requested a compatible version. " + + "Accept=" + + acceptHeader(null) + + ", Content-Type=" + + contentTypeHeader(CURRENT_VERSION) + ) + ); + + e = expectThrows( + ElasticsearchStatusException.class, + () -> requestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyPresent()) + ); + assertThat( + e.getMessage(), + equalTo( + "A compatible version is required on both Content-Type and Accept headers " + + "if either one has requested a compatible version and the compatible versions must match. " + + "Accept=" + + acceptHeader(null) + + ", Content-Type=" + + contentTypeHeader(PREVIOUS_VERSION) + ) + ); + + // tests when body NOT present and one of the headers missing + assertThat(requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(null), bodyNotPresent()), isCompatible()); + + assertThat(requestWith(acceptHeader(CURRENT_VERSION), contentTypeHeader(null), bodyNotPresent()), not(isCompatible())); + + // body not present - accept header is missing - it will default to Current version. Version on content type is ignored + assertThat(requestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()), not(isCompatible())); + + assertThat(requestWith(acceptHeader(null), contentTypeHeader(CURRENT_VERSION), bodyNotPresent()), not(isCompatible())); + + assertThat(requestWith(acceptHeader(null), contentTypeHeader(null), bodyNotPresent()), not(isCompatible())); + + // Accept header = application/json means current version. If body is provided then accept and content-Type should be the same + assertThat(requestWith(acceptHeader("application/json"), contentTypeHeader(null), bodyNotPresent()), not(isCompatible())); + + assertThat( + requestWith(acceptHeader("application/json"), contentTypeHeader("application/json"), bodyPresent()), + not(isCompatible()) + ); + + assertThat(requestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyPresent()), not(isCompatible())); + } + + public void testObsoleteVersion() { + ElasticsearchStatusException e = expectThrows( + ElasticsearchStatusException.class, + () -> requestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(OBSOLETE_VERSION), bodyPresent()) + ); + assertThat( + e.getMessage(), + equalTo( + "Accept version must be either version " + + CURRENT_VERSION + + " or " + + PREVIOUS_VERSION + + ", but found " + + OBSOLETE_VERSION + + ". " + + "Accept=" + + acceptHeader(OBSOLETE_VERSION) + ) + ); + + e = expectThrows( + ElasticsearchStatusException.class, + () -> requestWith(acceptHeader(OBSOLETE_VERSION), contentTypeHeader(null), bodyNotPresent()) + ); + assertThat( + e.getMessage(), + equalTo( + "Accept version must be either version " + + CURRENT_VERSION + + " or " + + PREVIOUS_VERSION + + ", but found " + + OBSOLETE_VERSION + + ". " + + "Accept=" + + acceptHeader(OBSOLETE_VERSION) + ) + ); + + e = expectThrows( + ElasticsearchStatusException.class, + () -> requestWith(acceptHeader(PREVIOUS_VERSION), contentTypeHeader(OBSOLETE_VERSION), bodyPresent()) + ); + assertThat( + e.getMessage(), + equalTo( + "Content-Type version must be either version " + + CURRENT_VERSION + + " or " + + PREVIOUS_VERSION + + ", but found " + + OBSOLETE_VERSION + + ". " + + "Content-Type=" + + contentTypeHeader(OBSOLETE_VERSION) + ) + ); + } + + public void testMediaTypeCombinations() { + // body not present - ignore content-type + assertThat(requestWith(acceptHeader(null), contentTypeHeader(PREVIOUS_VERSION), bodyNotPresent()), not(isCompatible())); + + assertThat(requestWith(acceptHeader(null), contentTypeHeader("application/json"), bodyNotPresent()), not(isCompatible())); + + assertThat(requestWith(acceptHeader("*/*"), contentTypeHeader("application/json"), bodyNotPresent()), not(isCompatible())); + + // this is for instance used by SQL + assertThat( + requestWith(acceptHeader("application/json"), contentTypeHeader("application/cbor"), bodyPresent()), + not(isCompatible()) + ); + + assertThat( + requestWith( + acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"), + contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=7"), + bodyPresent() + ), + isCompatible() + ); + + // different versions on different media types + expectThrows( + ElasticsearchStatusException.class, + () -> requestWith( + acceptHeader("application/vnd.elasticsearch+json;compatible-with=7"), + contentTypeHeader("application/vnd.elasticsearch+cbor;compatible-with=8"), + bodyPresent() + ) + ); + } + + public void testTextMediaTypes() { + assertThat( + requestWith(acceptHeader("text/tab-separated-values"), contentTypeHeader("application/json"), bodyNotPresent()), + not(isCompatible()) + ); + + assertThat(requestWith(acceptHeader("text/plain"), contentTypeHeader("application/json"), bodyNotPresent()), not(isCompatible())); + + assertThat(requestWith(acceptHeader("text/csv"), contentTypeHeader("application/json"), bodyNotPresent()), not(isCompatible())); + + // versioned + assertThat( + requestWith( + acceptHeader("text/vnd.elasticsearch+tab-separated-values;compatible-with=7"), + contentTypeHeader(7), + bodyNotPresent() + ), + isCompatible() + ); + + assertThat( + requestWith(acceptHeader("text/vnd.elasticsearch+plain;compatible-with=7"), contentTypeHeader(7), bodyNotPresent()), + isCompatible() + ); + + assertThat( + requestWith(acceptHeader("text/vnd.elasticsearch+csv;compatible-with=7"), contentTypeHeader(7), bodyNotPresent()), + isCompatible() + ); + } + + public void testVersionParsing() { + byte version = randomNonNegativeByte(); + assertThat( + CompatibleVersionHelper.parseVersion( + ParsedMediaType.parseMediaType("application/vnd.elasticsearch+json;compatible-with=" + version) + ), + equalTo(version) + ); + assertThat( + CompatibleVersionHelper.parseVersion( + ParsedMediaType.parseMediaType("application/vnd.elasticsearch+cbor;compatible-with=" + version) + ), + equalTo(version) + ); + assertThat( + CompatibleVersionHelper.parseVersion( + ParsedMediaType.parseMediaType("application/vnd.elasticsearch+smile;compatible-with=" + version) + ), + equalTo(version) + ); + assertThat( + CompatibleVersionHelper.parseVersion( + ParsedMediaType.parseMediaType("application/vnd.elasticsearch+x-ndjson;compatible-with=" + version) + ), + equalTo(version) + ); + assertThat(CompatibleVersionHelper.parseVersion(ParsedMediaType.parseMediaType("application/json")), nullValue()); + + assertThat( + CompatibleVersionHelper.parseVersion( + ParsedMediaType.parseMediaType("APPLICATION/VND.ELASTICSEARCH+JSON;COMPATIBLE-WITH=" + version) + ), + equalTo(version) + ); + assertThat(CompatibleVersionHelper.parseVersion(ParsedMediaType.parseMediaType("APPLICATION/JSON")), nullValue()); + + assertThat(CompatibleVersionHelper.parseVersion(ParsedMediaType.parseMediaType("application/json; sth=123")), is(nullValue())); + + } + + private Matcher isCompatible() { + return requestHasVersion(PREVIOUS_VERSION); + } + + private Matcher requestHasVersion(int version) { + return ElasticsearchMatchers.HasPropertyLambdaMatcher.hasProperty(v -> (int) v.major, equalTo(version)); + } + + private String bodyNotPresent() { + return ""; + } + + private String bodyPresent() { + return "some body"; + } + + private String contentTypeHeader(int version) { + return mediaType(String.valueOf(version)); + } + + private String acceptHeader(int version) { + return mediaType(String.valueOf(version)); + } + + private String acceptHeader(String value) { + return value; + } + + private String contentTypeHeader(String value) { + return value; + } + + private String mediaType(String version) { + if (version != null) { + return "application/vnd.elasticsearch+json;compatible-with=" + version; + } + return null; + } + + private Version requestWith(String accept, String contentType, String body) { + ParsedMediaType parsedAccept = ParsedMediaType.parseMediaType(accept); + ParsedMediaType parsedContentType = ParsedMediaType.parseMediaType(contentType); + return CompatibleVersionHelper.getCompatibleVersion(parsedAccept, parsedContentType, body.isEmpty() == false); + } + +} From bda3abca90286bc55ef46a646929d014aa14fa91 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 15 Feb 2021 15:33:49 +0000 Subject: [PATCH 35/43] Permit listing a nonexistent FsBlobContainer --- .../common/blobstore/fs/FsBlobContainer.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java index 893b969d0b2f2..54363b945b1e1 100644 --- a/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java +++ b/server/src/main/java/org/elasticsearch/common/blobstore/fs/FsBlobContainer.java @@ -37,6 +37,7 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicLong; @@ -88,7 +89,7 @@ public Map listBlobsByPrefix(String blobNamePrefix) throws Map builder = new HashMap<>(); blobNamePrefix = blobNamePrefix == null ? "" : blobNamePrefix; - try (DirectoryStream stream = Files.newDirectoryStream(path, blobNamePrefix + "*")) { + try (DirectoryStream stream = newDirectoryStreamIfFound(blobNamePrefix)) { for (Path file : stream) { final BasicFileAttributes attrs; try { @@ -105,6 +106,34 @@ public Map listBlobsByPrefix(String blobNamePrefix) throws return unmodifiableMap(builder); } + private DirectoryStream newDirectoryStreamIfFound(String blobNamePrefix) throws IOException { + try { + return Files.newDirectoryStream(path, blobNamePrefix + "*"); + } catch (FileNotFoundException | NoSuchFileException e) { + // a nonexistent directory contains no blobs + return new DirectoryStream<>() { + @Override + public Iterator iterator() { + return new Iterator<>() { + @Override + public boolean hasNext() { + return false; + } + + @Override + public Path next() { + return null; + } + }; + } + + @Override + public void close() { + } + }; + } + } + @Override public DeleteResult delete() throws IOException { final AtomicLong filesDeleted = new AtomicLong(0L); From 622fe24d04afe2f246d55eb1680c10cf1245f17f Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 15 Feb 2021 15:51:06 +0000 Subject: [PATCH 36/43] No-action TODOs --- .../repositories/blobstore/testkit/BlobAnalyzeAction.java | 2 +- .../repositories/blobstore/testkit/RandomBlobContent.java | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java index ba700a5c9134f..258dd8ec41a27 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java @@ -484,7 +484,7 @@ private void onReadsComplete(Collection responses, WriteDetails wr if (request.readEarly) { nodeFailure = null; // "not found" is legitimate iff we tried to read it before the write completed } else if (request.writeAndOverwrite) { - nodeFailure = null; // overwrites surprisingly not necessarily atomic; TODO are we ok with this?? + nodeFailure = null; // overwrites surprisingly not necessarily atomic, e.g. in a FsBlobContainer } else { nodeFailure = new RepositoryVerificationException( request.getRepositoryName(), diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java index 730b514a64e78..aa9125f214f58 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RandomBlobContent.java @@ -30,10 +30,9 @@ class RandomBlobContent { * @param repositoryName The name of the repository being tested, for use in exception messages. * @param seed RNG seed to use for its contents. * @param isCancelledSupplier Predicate that causes reads to throw a {@link RepositoryVerificationException}, allowing for fast failure - * on cancellation. TODO does this actually cancel blob writes gracefully enough? + * on cancellation. * @param onLastRead Runs when a {@code read()} call returns the last byte of the file, or on {@code close()} if the file was not fully * read. Only runs once even if the last byte is read multiple times using {@code mark()} and {@code reset()}. - * TODO does this happen just before completion or do some SDKs read the stream more than once (e.g. checksum first)? */ RandomBlobContent(String repositoryName, long seed, BooleanSupplier isCancelledSupplier, Runnable onLastRead) { this.repositoryName = repositoryName; From 4cae5105db36e68964e4d33262155b2bc703e658 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 16 Feb 2021 10:35:29 +0000 Subject: [PATCH 37/43] Docs improvements --- .../apis/repo-analysis-api.asciidoc | 31 +++++++++++-------- .../testkit/RepositoryAnalyzeAction.java | 6 ++-- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc index af9fe3ac8bdbe..8745570eae05f 100644 --- a/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/repo-analysis-api.asciidoc @@ -41,8 +41,8 @@ systems behave incorrectly, or perform poorly, especially when accessed concurrently by multiple clients as the nodes of an {es} cluster do. The Repository analysis API performs a collection of read and write operations -on your repository which are specially designed to detect incorrect behaviour -and to measure the performance characteristics of your storage system. +on your repository which are designed to detect incorrect behaviour and to +measure the performance characteristics of your storage system. The default values for the parameters to this API are deliberately low to reduce the impact of running an analysis. A realistic experiment should set @@ -64,14 +64,14 @@ been cleaned up correctly. If there is still leftover data at the specified location then you should manually remove it. If the connection from your client to {es} is closed while the client is -waiting for the result of the analysis then the test is cancelled. Since an -analysis takes a long time to complete, you may need to configure your client -to wait for longer than usual for a response. On cancellation the analysis -attempts to clean up the data it was writing, but it may not be able to remove -it all. The path to the leftover data is recorded in the {es} logs. You should -verify yourself that this location has been cleaned up correctly. If there is -still leftover data at the specified location then you should manually remove -it. +waiting for the result of the analysis then the test is cancelled. Some clients +are configured to close their connection if no response is received within a +certain timeout. An analysis takes a long time to complete so you may need to +relax any such client-side timeouts. On cancellation the analysis attempts to +clean up the data it was writing, but it may not be able to remove it all. The +path to the leftover data is recorded in the {es} logs. You should verify +yourself that this location has been cleaned up correctly. If there is still +leftover data at the specified location then you should manually remove it. If the analysis is successful then it detected no incorrect behaviour, but this does not mean that correct behaviour is guaranteed. The analysis attempts to @@ -99,12 +99,17 @@ these systems. Analyses respect the repository settings the cluster setting `indices.recovery.max_bytes_per_sec` which you can use to limit the bandwidth they consume. +NOTE: This API is intended for exploratory use by humans. You should expect the +request parameters and the response format to vary in future versions. + +NOTE: This API may not work correctly in a mixed-version cluster. + ==== Implementation details NOTE: This section of documentation describes how the Repository analysis API works in this version of {es}, but you should expect the implementation to vary -between versions. The response format exposes details of the implementation so -it may also be different in newer versions. +between versions. The request parameters and response format depend on details +of the implementation so may also be different in newer versions. The analysis comprises a number of blob-level tasks, as set by the `blob_count` parameter. The blob-level tasks are distributed over the data and @@ -274,7 +279,7 @@ The probability of performing rare actions during the test. Equal to the The path in the repository under which all the blobs were written during the test. -`bugs_detected`:: +`issues_detected`:: (list) A list of correctness issues detected, which will be empty if the API succeeded. Included to emphasize that a successful response does not guarantee diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index 2ff23cf211cae..3ed177437eb7f 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -987,9 +987,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("rare_action_probability", rareActionProbability); builder.field("blob_path", blobPath); - builder.startArray("bugs_detected"); - // nothing to report here, if we detected a bug then we would have thrown an exception, but we include this to emphasise - // that we are only detecting bugs, not guaranteeing their absence + builder.startArray("issues_detected"); + // nothing to report here, if we detected an issue then we would have thrown an exception, but we include this to emphasise + // that we are only detecting issues, not guaranteeing their absence builder.endArray(); builder.field("summary", summary); From 12ccaba93a08e2ed5363f2c980ce260e059cf409 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 16 Feb 2021 10:38:10 +0000 Subject: [PATCH 38/43] ActionRunnable just to be sure that exceptions are caught --- .../blobstore/testkit/RepositoryAnalyzeAction.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index 3ed177437eb7f..f1e651daf3482 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -18,6 +18,7 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.HandledTransportAction; @@ -521,13 +522,13 @@ private BlobContainer getBlobContainer() { private void onWorkerCompletion() { if (workerCountdown.countDown()) { - transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT).execute(() -> { + transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT).execute(ActionRunnable.run(listener, () -> { final long listingStartTimeNanos = System.nanoTime(); ensureConsistentListing(); final long deleteStartTimeNanos = System.nanoTime(); deleteContainer(); sendResponse(listingStartTimeNanos, deleteStartTimeNanos); - }); + })); } } From 6308cccfb10727224a95d708fb7e70bda0f0db1b Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 16 Feb 2021 11:14:11 +0000 Subject: [PATCH 39/43] Human-readability --- .../rest-api-spec/test/10_analyze.yml | 62 ++++++++++++++----- .../blobstore/testkit/BlobAnalyzeAction.java | 17 ++--- .../testkit/RepositoryAnalyzeAction.java | 10 +-- .../testkit/RepositoryPerformanceSummary.java | 19 +++--- .../testkit/RestRepositoryAnalyzeAction.java | 10 ++- .../testkit/SnapshotRepositoryTestKit.java | 13 ++++ 6 files changed, 95 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyze.yml b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyze.yml index 18b30eeecf12d..1b7df0cc5fbc0 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyze.yml +++ b/x-pack/plugin/snapshot-repo-test-kit/qa/rest/src/test/resources/rest-api-spec/test/10_analyze.yml @@ -60,22 +60,34 @@ setup: - match: { early_read_node_count: 1 } - match: { rare_action_probability: 0.01 } - match: { max_blob_size: 1mb } + - match: { max_blob_size_bytes: 1048576 } - match: { max_total_data_size: 5mb } + - match: { max_total_data_size_bytes: 5242880 } - is_true: seed - is_true: blob_path - is_false: details - - gte: { listing_nanos: 0} - - gte: { delete_nanos: 0} + - is_true: listing_elapsed + - is_true: delete_elapsed + - gte: { listing_elapsed_nanos: 0} + - gte: { delete_elapsed_nanos: 0} - match: { summary.write.count: 10} - - gte: { summary.write.total_bytes: 0} + - gte: { summary.write.total_size_bytes: 0} + - is_true: summary.write.total_size - gte: { summary.write.total_throttled_nanos: 0} + - is_true: summary.write.total_throttled - gte: { summary.write.total_elapsed_nanos: 0} + - is_true: summary.write.total_elapsed - gte: { summary.read.count: 10} - - gte: { summary.read.total_bytes: 0} + - gte: { summary.read.total_size_bytes: 0} + - is_true: summary.read.total_size - gte: { summary.read.total_wait_nanos: 0} + - is_true: summary.read.total_wait - gte: { summary.read.max_wait_nanos: 0} + - is_true: summary.read.max_wait - gte: { summary.read.total_throttled_nanos: 0} + - is_true: summary.read.total_throttled - gte: { summary.read.total_elapsed_nanos: 0} + - is_true: summary.read.total_elapsed --- "Analysis with details": @@ -103,15 +115,33 @@ setup: - is_true: seed - is_true: blob_path - is_true: details - - gte: { listing_nanos: 0} - - gte: { delete_nanos: 0} - - match: { summary.write.count: 10} - - gte: { summary.write.total_bytes: 0} - - gte: { summary.write.total_throttled_nanos: 0} - - gte: { summary.write.total_elapsed_nanos: 0} - - gte: { summary.read.count: 10} - - gte: { summary.read.total_bytes: 0} - - gte: { summary.read.total_wait_nanos: 0} - - gte: { summary.read.max_wait_nanos: 0} - - gte: { summary.read.total_throttled_nanos: 0} - - gte: { summary.read.total_elapsed_nanos: 0} + - is_true: listing_elapsed + - is_true: delete_elapsed + - gte: { listing_elapsed_nanos: 0} + - gte: { delete_elapsed_nanos: 0} + +--- +"Analysis with ?human=false": + - skip: + version: "- 7.99.99" + reason: "introduced in 8.0" + + - do: + snapshot.repository_analyze: + repository: test_repo + blob_count: 10 + concurrency: 5 + max_blob_size: 1mb + detailed: false + human: false + + - is_false: listing_elapsed + - is_false: delete_elapsed + - is_false: summary.write.total_size + - is_false: summary.write.total_throttled + - is_false: summary.write.total_elapsed + - is_false: summary.read.total_size + - is_false: summary.read.total_wait + - is_false: summary.read.max_wait + - is_false: summary.read.total_throttled + - is_false: summary.read.total_elapsed diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java index 258dd8ec41a27..a4c778d2aa150 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/BlobAnalyzeAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -56,6 +57,8 @@ import java.util.function.LongPredicate; import java.util.stream.Collectors; +import static org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit.humanReadableNanos; + /** * Action which instructs a node to write a blob to the blob store and verify that it can be read correctly by other nodes. The other nodes * may read the whole blob or just a range; we verify the data that is read by checksum using {@link GetBlobChecksumAction}. @@ -807,7 +810,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject("blob"); builder.field("name", blobName); - builder.field("size", blobLength); + builder.humanReadableField("size_bytes", "size", new ByteSizeValue(blobLength)); builder.field("read_start", checksumStart); builder.field("read_end", checksumEnd); builder.field("read_early", readEarly); @@ -819,11 +822,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("name", nodeName); builder.endObject(); - builder.field("write_elapsed_nanos", writeElapsedNanos); + humanReadableNanos(builder, "write_elapsed_nanos", "write_elapsed", writeElapsedNanos); if (overwrite) { - builder.field("overwrite_elapsed_nanos", overwriteElapsedNanos); + humanReadableNanos(builder, "overwrite_elapsed_nanos", "overwrite_elapsed", overwriteElapsedNanos); } - builder.field("write_throttled_nanos", writeThrottledNanos); + humanReadableNanos(builder, "write_throttled_nanos", "write_throttled", writeThrottledNanos); builder.startArray("reads"); for (ReadDetail readDetail : readDetails) { @@ -922,9 +925,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("found", false); } else { builder.field("found", true); - builder.field("first_byte_nanos", firstByteNanos); - builder.field("elapsed_nanos", elapsedNanos); - builder.field("throttled_nanos", throttleNanos); + humanReadableNanos(builder, "first_byte_time_nanos", "first_byte_time", firstByteNanos); + humanReadableNanos(builder, "elapsed_nanos", "elapsed", elapsedNanos); + humanReadableNanos(builder, "throttled_nanos", "throttled", throttleNanos); } builder.endObject(); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index f1e651daf3482..c18e35ecede35 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -70,6 +70,8 @@ import java.util.function.LongSupplier; import java.util.stream.IntStream; +import static org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit.humanReadableNanos; + /** * Action which distributes a bunch of {@link BlobAnalyzeAction}s over the nodes in the cluster, with limited concurrency, and collects * the results. Tries to fail fast by cancelling everything if any child task fails, or the timeout is reached, to avoid consuming @@ -982,8 +984,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("concurrency", concurrency); builder.field("read_node_count", readNodeCount); builder.field("early_read_node_count", earlyReadNodeCount); - builder.field("max_blob_size", maxBlobSize); - builder.field("max_total_data_size", maxTotalDataSize); + builder.humanReadableField("max_blob_size_bytes", "max_blob_size", maxBlobSize); + builder.humanReadableField("max_total_data_size_bytes", "max_total_data_size", maxTotalDataSize); builder.field("seed", seed); builder.field("rare_action_probability", rareActionProbability); builder.field("blob_path", blobPath); @@ -1003,8 +1005,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.endArray(); } - builder.field("listing_nanos", listingTimeNanos); - builder.field("delete_nanos", deleteTimeNanos); + humanReadableNanos(builder, "listing_elapsed_nanos", "listing_elapsed", listingTimeNanos); + humanReadableNanos(builder, "delete_elapsed_nanos", "delete_elapsed", deleteTimeNanos); builder.endObject(); return builder; diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java index 9459971b29b1f..8ac0090390c24 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryPerformanceSummary.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -17,6 +18,8 @@ import java.util.concurrent.atomic.LongAccumulator; import java.util.concurrent.atomic.LongAdder; +import static org.elasticsearch.repositories.blobstore.testkit.SnapshotRepositoryTestKit.humanReadableNanos; + public class RepositoryPerformanceSummary implements Writeable, ToXContentFragment { private final long writeCount; @@ -87,18 +90,18 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject("write"); builder.field("count", writeCount); - builder.field("total_bytes", writeBytes); - builder.field("total_throttled_nanos", writeThrottledNanos); - builder.field("total_elapsed_nanos", writeElapsedNanos); + builder.humanReadableField("total_size_bytes", "total_size", new ByteSizeValue(writeBytes)); + humanReadableNanos(builder, "total_throttled_nanos", "total_throttled", writeThrottledNanos); + humanReadableNanos(builder, "total_elapsed_nanos", "total_elapsed", writeElapsedNanos); builder.endObject(); builder.startObject("read"); builder.field("count", readCount); - builder.field("total_bytes", readBytes); - builder.field("total_wait_nanos", readWaitNanos); - builder.field("max_wait_nanos", maxReadWaitNanos); - builder.field("total_throttled_nanos", readThrottledNanos); - builder.field("total_elapsed_nanos", readElapsedNanos); + builder.humanReadableField("total_size_bytes", "total_size", new ByteSizeValue(readBytes)); + humanReadableNanos(builder, "total_wait_nanos", "total_wait", readWaitNanos); + humanReadableNanos(builder, "max_wait_nanos", "max_wait", maxReadWaitNanos); + humanReadableNanos(builder, "total_throttled_nanos", "total_throttled", readThrottledNanos); + humanReadableNanos(builder, "total_elapsed_nanos", "total_elapsed", readElapsedNanos); builder.endObject(); builder.endObject(); diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java index d695eb7c97aa0..53cfedce1a262 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RestRepositoryAnalyzeAction.java @@ -8,8 +8,10 @@ package org.elasticsearch.repositories.blobstore.testkit; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.action.RestCancellableNodeClient; import org.elasticsearch.rest.action.RestStatusToXContentListener; @@ -54,7 +56,13 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> cancelClient.execute( RepositoryAnalyzeAction.INSTANCE, analyzeRepositoryRequest, - new RestStatusToXContentListener<>(channel) + new RestStatusToXContentListener<>(channel) { + @Override + public RestResponse buildResponse(RepositoryAnalyzeAction.Response response, XContentBuilder builder) throws Exception { + builder.humanReadable(request.paramAsBoolean("human", true)); + return super.buildResponse(response, builder); + } + } ); } } diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java index ffc84078893aa..7dea49014d1e2 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/SnapshotRepositoryTestKit.java @@ -15,11 +15,14 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; +import java.io.IOException; import java.util.List; import java.util.function.Supplier; @@ -46,4 +49,14 @@ public List getRestHandlers( ) { return List.of(new RestRepositoryAnalyzeAction()); } + + static void humanReadableNanos(XContentBuilder builder, String rawFieldName, String readableFieldName, long nanos) throws IOException { + assert rawFieldName.equals(readableFieldName) == false : rawFieldName + " vs " + readableFieldName; + + if (builder.humanReadable()) { + builder.field(readableFieldName, TimeValue.timeValueNanos(nanos).toHumanReadableString(2)); + } + + builder.field(rawFieldName, nanos); + } } From f10e51cd6cb5b546900adef5d01930e1a62e8c0f Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 16 Feb 2021 11:25:13 +0000 Subject: [PATCH 40/43] Javadoc --- .../blobstore/testkit/RepositoryAnalyzeAction.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index c18e35ecede35..44c20a3776c02 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -188,6 +188,11 @@ private static List getSnapshotNodes(DiscoveryNodes discoveryNode return nodes; } + /** + * Compute a collection of blob sizes which respects the blob count and the limits on the size of each blob and the total size to write. + * Tries its best to achieve an even spread of different-sized blobs to try and exercise the various size-based code paths well. See + * the corresponding unit tests for examples of its output. + */ // Exposed for tests static List getBlobSizes(Request request) { From ffee1a7d42a9fd69c2dd2418e124c2ef499279e1 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 16 Feb 2021 11:37:32 +0000 Subject: [PATCH 41/43] Close input stream used for reading --- .../testkit/GetBlobChecksumAction.java | 64 ++++++++++++------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java index 08dbd6f688c09..d8fe5a2643dfc 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/GetBlobChecksumAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryVerificationException; @@ -115,36 +116,53 @@ protected void doExecute(Task task, Request request, ActionListener li final long startTimeNanos = System.nanoTime(); long firstByteNanos = startTimeNanos; - while (true) { - final int readSize; - try { - readSize = throttledInputStream.read(buffer, 0, buffer.length); - } catch (IOException e) { - logger.warn("exception while read blob for [{}]", request); - listener.onFailure(e); - return; - } + boolean success = false; + try { + while (true) { + final int readSize; + try { + readSize = throttledInputStream.read(buffer, 0, buffer.length); + } catch (IOException e) { + logger.warn("exception while read blob for [{}]", request); + listener.onFailure(e); + return; + } - if (readSize == -1) { - break; - } + if (readSize == -1) { + break; + } + + if (readSize > 0) { + if (bytesRead == 0L) { + firstByteNanos = System.nanoTime(); + } - if (readSize > 0) { - if (bytesRead == 0L) { - firstByteNanos = System.nanoTime(); + crc32.update(buffer, 0, readSize); + bytesRead += readSize; } - crc32.update(buffer, 0, readSize); - bytesRead += readSize; + if (cancellableTask.isCancelled()) { + throw new RepositoryVerificationException( + request.repositoryName, + "cancelled [" + request.getDescription() + "] after reading [" + bytesRead + "] bytes" + ); + } } - - if (cancellableTask.isCancelled()) { - throw new RepositoryVerificationException( - request.repositoryName, - "cancelled [" + request.getDescription() + "] after reading [" + bytesRead + "] bytes" - ); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(throttledInputStream); } } + try { + throttledInputStream.close(); + } catch (IOException e) { + throw new RepositoryVerificationException( + request.repositoryName, + "failed to close input stream when handling [" + request.getDescription() + "]", + e + ); + } final long endTimeNanos = System.nanoTime(); From b3c6beeea5bd952a58d287d9af70384f438011f4 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 16 Feb 2021 11:49:57 +0000 Subject: [PATCH 42/43] TODO also non-powers-of-two --- .../repositories/blobstore/testkit/RepositoryAnalyzeAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index 44c20a3776c02..d63ee0061cbde 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -217,6 +217,7 @@ static List getBlobSizes(Request request) { blobSizes.add(s); } blobSizes.add(maxBlobSize); + // TODO also use sizes that aren't a power of two // Try and form an even spread of blob sizes by accounting for as many evenly spreads repeats as possible up-front. final long evenSpreadSize = blobSizes.stream().mapToLong(l -> l).sum(); From d0887faa9cc9082ae7c1e7d203152a1c67d1e1a5 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 16 Feb 2021 11:52:26 +0000 Subject: [PATCH 43/43] Clearer comment --- .../repositories/blobstore/testkit/RepositoryAnalyzeAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java index d63ee0061cbde..a3158833637a9 100644 --- a/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java +++ b/x-pack/plugin/snapshot-repo-test-kit/src/main/java/org/elasticsearch/repositories/blobstore/testkit/RepositoryAnalyzeAction.java @@ -452,7 +452,7 @@ public void run() { for (int i = 0; i < request.getBlobCount(); i++) { final long targetLength = blobSizes.get(i); - final boolean smallBlob = targetLength <= Integer.MAX_VALUE; // we only use the non-atomic API for larger blobs + final boolean smallBlob = targetLength <= Integer.MAX_VALUE; // avoid the atomic API for larger blobs final VerifyBlobTask verifyBlobTask = new VerifyBlobTask( nodes.get(random.nextInt(nodes.size())), new BlobAnalyzeAction.Request(