Skip to content

Commit a819079

Browse files
fcofdezfcofdez
authored
Add integration tests for repository analyser test kit (#69316)
Relates #67247
1 parent 828fd3a commit a819079

File tree

13 files changed

+505
-1
lines changed

13 files changed

+505
-1
lines changed

‎test/fixtures/azure-fixture/docker-compose.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,12 @@ services:
2626
- ./testfixtures_shared/shared:/fixture/shared
2727
ports:
2828
- "8091"
29+
30+
azure-fixture-repository-test-kit:
31+
build:
32+
context: .
33+
dockerfile: Dockerfile
34+
volumes:
35+
- ./testfixtures_shared/shared:/fixture/shared
36+
ports:
37+
- "8091"

‎test/fixtures/gcs-fixture/docker-compose.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,15 @@ services:
4848
- ./testfixtures_shared/shared:/fixture/shared
4949
ports:
5050
- "80"
51+
gcs-fixture-repository-test-kit:
52+
build:
53+
context: .
54+
args:
55+
port: 80
56+
bucket: "bucket"
57+
token: "o/oauth2/token"
58+
dockerfile: Dockerfile
59+
volumes:
60+
- ./testfixtures_shared/shared:/fixture/shared
61+
ports:
62+
- "80"

‎test/fixtures/minio-fixture/docker-compose.yml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,14 @@ services:
2222
ports:
2323
- "9000"
2424
command: ["server", "/minio/data"]
25+
minio-fixture-repository-test-kit:
26+
build:
27+
context: .
28+
args:
29+
bucket: "bucket"
30+
accessKey: "s3_test_access_key"
31+
secretKey: "s3_test_secret_key"
32+
dockerfile: Dockerfile
33+
ports:
34+
- "9000"
35+
command: ["server", "/minio/data"]

‎test/fixtures/s3-fixture/docker-compose.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,22 @@ services:
4545
ports:
4646
- "80"
4747

48+
s3-fixture-repository-test-kit:
49+
build:
50+
context: .
51+
args:
52+
fixtureClass: fixture.s3.S3HttpFixture
53+
port: 80
54+
bucket: "bucket"
55+
basePath: "base_path"
56+
accessKey: "s3_test_access_key"
57+
dockerfile: Dockerfile
58+
volumes:
59+
- ./testfixtures_shared/shared:/fixture/shared
60+
- /Users/fran/test:/Users/fran/test
61+
ports:
62+
- "80"
63+
4864
s3-fixture-with-session-token:
4965
build:
5066
context: .

‎test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -243,7 +243,7 @@ public void handle(final HttpExchange exchange) throws IOException {
243243
deletes.append("<DeleteResult>");
244244
for (Iterator<Map.Entry<String, BytesReference>> iterator = blobs.entrySet().iterator(); iterator.hasNext(); ) {
245245
Map.Entry<String, BytesReference> blob = iterator.next();
246-
String key = blob.getKey().replace("/" + path + "/", "");
246+
String key = blob.getKey().replace("/" + bucket + "/", "");
247247
if (requestBody.contains("<Key>" + key + "</Key>")) {
248248
deletes.append("<Deleted><Key>").append(key).append("</Key></Deleted>");
249249
iterator.remove();

‎x-pack/plugin/snapshot-repo-test-kit/qa/azure/build.gradle

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import org.elasticsearch.gradle.info.BuildParams
9+
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
10+
11+
apply plugin: 'elasticsearch.standalone-rest-test'
12+
apply plugin: 'elasticsearch.rest-test'
13+
apply plugin: 'elasticsearch.rest-resources'
14+
15+
final Project fixture = project(':test:fixtures:azure-fixture')
16+
final Project repositoryPlugin = project(':plugins:repository-azure')
17+
18+
dependencies {
19+
testImplementation testArtifact(project(xpackModule('snapshot-repo-test-kit')))
20+
testImplementation repositoryPlugin
21+
}
22+
23+
restResources {
24+
restApi {
25+
includeCore 'indices', 'search', 'bulk', 'snapshot', 'nodes', '_common'
26+
includeXpack 'snapshot_repo_test_kit'
27+
}
28+
}
29+
30+
boolean useFixture = false
31+
String azureAccount = System.getenv("azure_storage_account")
32+
String azureKey = System.getenv("azure_storage_key")
33+
String azureContainer = System.getenv("azure_storage_container")
34+
String azureBasePath = System.getenv("azure_storage_base_path")
35+
String azureSasToken = System.getenv("azure_storage_sas_token")
36+
37+
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath && !azureSasToken) {
38+
azureAccount = 'azure_integration_test_account'
39+
azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
40+
azureContainer = 'container'
41+
azureBasePath = ''
42+
azureSasToken = ''
43+
useFixture = true
44+
45+
}
46+
47+
if (useFixture) {
48+
apply plugin: 'elasticsearch.test.fixtures'
49+
testFixtures.useFixture(fixture.path, 'azure-fixture-repository-test-kit')
50+
}
51+
52+
tasks.named("integTest").configure {
53+
systemProperty 'test.azure.container', azureContainer
54+
nonInputProperties.systemProperty 'test.azure.base_path', azureBasePath + "_repository_test_kit_tests_" + BuildParams.testSeed
55+
}
56+
57+
testClusters.matching { it.name == "integTest" }.configureEach {
58+
testDistribution = 'DEFAULT'
59+
plugin repositoryPlugin.path
60+
61+
keystore 'azure.client.repository_test_kit.account', azureAccount
62+
if (azureKey != null && azureKey.isEmpty() == false) {
63+
keystore 'azure.client.repository_test_kit.key', azureKey
64+
}
65+
if (azureSasToken != null && azureSasToken.isEmpty() == false) {
66+
keystore 'azure.client.repository_test_kit.sas_token', azureSasToken
67+
}
68+
69+
if (useFixture) {
70+
def fixtureAddress = { fixtureName ->
71+
assert useFixture: 'closure should not be used without a fixture'
72+
int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.8091"
73+
assert ephemeralPort > 0
74+
'127.0.0.1:' + ephemeralPort
75+
}
76+
setting 'azure.client.repository_test_kit.endpoint_suffix',
77+
{ "ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${-> fixtureAddress('azure-fixture-other')}/azure_integration_test_account" }, IGNORE_VALUE
78+
79+
} else {
80+
println "Using an external service to test " + project.name
81+
}
82+
}
83+
84+
tasks.register("azureThirdPartyTest") {
85+
dependsOn "integTest"
86+
}

‎x-pack/plugin/snapshot-repo-test-kit/qa/azure/src/test/java/org/elasticsearch/repositories/blobstore/testkit/AzureSnapshotRepoTestKitIT.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
package org.elasticsearch.repositories.blobstore.testkit;
8+
9+
import org.elasticsearch.common.settings.Settings;
10+
11+
import static org.hamcrest.Matchers.blankOrNullString;
12+
import static org.hamcrest.Matchers.not;
13+
14+
public class AzureSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestTestCase {
15+
16+
@Override
17+
protected String repositoryType() {
18+
return "azure";
19+
}
20+
21+
@Override
22+
protected Settings repositorySettings() {
23+
final String container = System.getProperty("test.azure.container");
24+
assertThat(container, not(blankOrNullString()));
25+
26+
final String basePath = System.getProperty("test.azure.base_path");
27+
assertThat(basePath, not(blankOrNullString()));
28+
29+
return Settings.builder().put("client", "repository_test_kit").put("container", container).put("base_path", basePath).build();
30+
}
31+
}

‎x-pack/plugin/snapshot-repo-test-kit/qa/gcs/build.gradle

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import org.elasticsearch.gradle.info.BuildParams
9+
import org.elasticsearch.gradle.MavenFilteringHack
10+
11+
import java.nio.file.Files
12+
import java.security.KeyPair
13+
import java.security.KeyPairGenerator
14+
15+
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
16+
17+
apply plugin: 'elasticsearch.standalone-rest-test'
18+
apply plugin: 'elasticsearch.rest-test'
19+
apply plugin: 'elasticsearch.rest-resources'
20+
21+
final Project fixture = project(':test:fixtures:gcs-fixture')
22+
final Project repositoryPlugin = project(':plugins:repository-gcs')
23+
24+
dependencies {
25+
testImplementation testArtifact(project(xpackModule('snapshot-repo-test-kit')))
26+
testImplementation repositoryPlugin
27+
}
28+
29+
restResources {
30+
restApi {
31+
includeCore 'indices', 'search', 'bulk', 'snapshot', 'nodes', '_common'
32+
includeXpack 'snapshot_repo_test_kit'
33+
}
34+
}
35+
36+
boolean useFixture = false
37+
38+
String gcsServiceAccount = System.getenv("google_storage_service_account")
39+
String gcsBucket = System.getenv("google_storage_bucket")
40+
String gcsBasePath = System.getenv("google_storage_base_path")
41+
42+
File serviceAccountFile = null
43+
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
44+
serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
45+
gcsBucket = 'bucket'
46+
gcsBasePath = 'integration_test'
47+
useFixture = true
48+
} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
49+
throw new IllegalArgumentException("not all options specified to run tests against external GCS service are present")
50+
} else {
51+
serviceAccountFile = new File(gcsServiceAccount)
52+
}
53+
54+
tasks.register("createServiceAccountFile") {
55+
doLast {
56+
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
57+
keyPairGenerator.initialize(2048)
58+
KeyPair keyPair = keyPairGenerator.generateKeyPair()
59+
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
60+
61+
serviceAccountFile.parentFile.mkdirs()
62+
serviceAccountFile.setText("{\n" +
63+
' "type": "service_account",\n' +
64+
' "project_id": "integration_test",\n' +
65+
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
66+
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
67+
' "client_email": "[email protected]",\n' +
68+
' "client_id": "123456789101112130594"\n' +
69+
'}', 'UTF-8')
70+
}
71+
}
72+
73+
def fixtureAddress = { f ->
74+
assert useFixture: 'closure should not be used without a fixture'
75+
int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${f}.tcp.80"
76+
assert ephemeralPort > 0
77+
'http://127.0.0.1:' + ephemeralPort
78+
}
79+
80+
Map<String, Object> expansions = [
81+
'bucket' : gcsBucket,
82+
'base_path': gcsBasePath + "_integration_tests"
83+
]
84+
85+
if (useFixture) {
86+
apply plugin: 'elasticsearch.test.fixtures'
87+
testFixtures.useFixture(fixture.path, 'gcs-fixture-repository-test-kit')
88+
}
89+
90+
tasks.named("integTest").configure {
91+
systemProperty 'test.gcs.bucket', gcsBucket
92+
nonInputProperties.systemProperty 'test.gcs.base_path', gcsBasePath + "_repository_test_kit_tests" + BuildParams.testSeed
93+
94+
if (useFixture) {
95+
dependsOn "createServiceAccountFile"
96+
}
97+
}
98+
99+
testClusters.matching { it.name == "integTest" }.configureEach {
100+
testDistribution = 'DEFAULT'
101+
plugin repositoryPlugin.path
102+
103+
keystore 'gcs.client.repository_test_kit.credentials_file', serviceAccountFile, IGNORE_VALUE
104+
if (useFixture) {
105+
/* Use a closure on the string to delay evaluation until tests are executed */
106+
setting 'gcs.client.repository_test_kit.endpoint', { "${-> fixtureAddress('gcs-fixture-repository-test-kit')}" }, IGNORE_VALUE
107+
setting 'gcs.client.repository_test_kit.token_uri', { "${-> fixtureAddress('gcs-fixture-repository-test-kit')}/o/oauth2/token" }, IGNORE_VALUE
108+
} else {
109+
println "Using an external service to test " + project.name
110+
}
111+
}
112+
113+
tasks.register("gcsThirdPartyTest") {
114+
dependsOn "integTest"
115+
}

‎x-pack/plugin/snapshot-repo-test-kit/qa/gcs/src/test/java/org/elasticsearch/repositories/blobstore/testkit/GCSSnapshotRepoTestKitIT.java

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
package org.elasticsearch.repositories.blobstore.testkit;
8+
9+
import org.elasticsearch.common.settings.Settings;
10+
11+
import static org.hamcrest.Matchers.blankOrNullString;
12+
import static org.hamcrest.Matchers.not;
13+
14+
public class GCSSnapshotRepoTestKitIT extends AbstractSnapshotRepoTestKitRestTestCase {
15+
16+
@Override
17+
protected String repositoryType() {
18+
return "gcs";
19+
}
20+
21+
@Override
22+
protected Settings repositorySettings() {
23+
final String bucket = System.getProperty("test.gcs.bucket");
24+
assertThat(bucket, not(blankOrNullString()));
25+
26+
final String basePath = System.getProperty("test.gcs.base_path");
27+
assertThat(basePath, not(blankOrNullString()));
28+
29+
return Settings.builder().put("client", "repository_test_kit").put("bucket", bucket).put("base_path", basePath).build();
30+
}
31+
}

‎x-pack/plugin/snapshot-repo-test-kit/qa/minio/build.gradle

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
9+
10+
apply plugin: 'elasticsearch.standalone-rest-test'
11+
apply plugin: 'elasticsearch.rest-test'
12+
apply plugin: 'elasticsearch.test.fixtures'
13+
apply plugin: 'elasticsearch.rest-resources'
14+
15+
final Project fixture = project(':test:fixtures:minio-fixture')
16+
final Project repositoryPlugin = project(':plugins:repository-s3')
17+
18+
dependencies {
19+
testImplementation testArtifact(project(xpackModule('snapshot-repo-test-kit')))
20+
testImplementation repositoryPlugin
21+
}
22+
23+
restResources {
24+
restApi {
25+
includeCore 'indices', 'search', 'bulk', 'snapshot', 'nodes', '_common'
26+
includeXpack 'snapshot_repo_test_kit'
27+
}
28+
}
29+
30+
testFixtures.useFixture(fixture.path, 'minio-fixture-repository-test-kit')
31+
def fixtureAddress = {
32+
int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.minio-fixture-repository-test-kit.tcp.9000"
33+
assert ephemeralPort > 0
34+
'127.0.0.1:' + ephemeralPort
35+
}
36+
37+
tasks.named("integTest").configure {
38+
systemProperty 'test.minio.bucket', 'bucket'
39+
systemProperty 'test.minio.base_path', 'repository_test_kit_tests'
40+
}
41+
42+
testClusters.matching { it.name == "integTest" }.configureEach {
43+
testDistribution = 'DEFAULT'
44+
plugin repositoryPlugin.path
45+
46+
keystore 's3.client.repository_test_kit.access_key', 's3_test_access_key'
47+
keystore 's3.client.repository_test_kit.secret_key', 's3_test_secret_key'
48+
setting 's3.client.repository_test_kit.protocol', 'http'
49+
setting 's3.client.repository_test_kit.endpoint', { "${-> fixtureAddress()}" }, IGNORE_VALUE
50+
}
51+

0 commit comments

Comments
 (0)