Skip to content

Commit 60b18b5

Browse files
tlrxtlrx
authored
Move repository-gcs fixture test to QA project (#30208)
Similarly to what has been done in for the repository-s3 plugin, this commit moves the fixture test into a dedicated repository-gcs/qa/google-cloud-storage project. It also exposes some environment variables which allows to execute the integration tests against the real Google Cloud Storage service. When the environment variables are not defined, the integration tests are executed using the fixture added in #28788. Related to #29349.
1 parent a6624bb commit 60b18b5

File tree

8 files changed

+333
-226
lines changed

8 files changed

+333
-226
lines changed

‎plugins/repository-gcs/build.gradle

+4-52
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,3 @@
1-
import org.elasticsearch.gradle.test.AntFixture
2-
3-
import java.security.KeyPair
4-
import java.security.KeyPairGenerator
5-
61
/*
72
* Licensed to Elasticsearch under one or more contributor
83
* license agreements. See the NOTICE file distributed with
@@ -58,50 +53,7 @@ thirdPartyAudit.excludes = [
5853
'org.apache.log.Logger',
5954
]
6055

61-
forbiddenApisTest {
62-
// we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
63-
bundledSignatures -= 'jdk-non-portable'
64-
bundledSignatures += 'jdk-internal'
65-
}
66-
67-
/** A task to start the GoogleCloudStorageFixture which emulates a Google Cloud Storage service **/
68-
task googleCloudStorageFixture(type: AntFixture) {
69-
dependsOn compileTestJava
70-
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
71-
executable = new File(project.runtimeJavaHome, 'bin/java')
72-
args 'org.elasticsearch.repositories.gcs.GoogleCloudStorageFixture', baseDir, 'bucket_test'
73-
}
74-
75-
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
76-
File serviceAccountFile = new File(project.buildDir, "generated-resources/service_account_test.json")
77-
task createServiceAccountFile() {
78-
dependsOn googleCloudStorageFixture
79-
doLast {
80-
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
81-
keyPairGenerator.initialize(1024)
82-
KeyPair keyPair = keyPairGenerator.generateKeyPair()
83-
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
84-
85-
serviceAccountFile.parentFile.mkdirs()
86-
serviceAccountFile.setText("{\n" +
87-
' "type": "service_account",\n' +
88-
' "project_id": "integration_test",\n' +
89-
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
90-
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
91-
' "client_email": "[email protected]",\n' +
92-
' "client_id": "123456789101112130594",\n' +
93-
" \"auth_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/auth\",\n" +
94-
" \"token_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token\",\n" +
95-
' "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",\n' +
96-
' "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/integration_test%40appspot.gserviceaccount.com"\n' +
97-
'}', 'UTF-8')
98-
}
99-
}
100-
101-
integTestCluster {
102-
dependsOn createServiceAccountFile, googleCloudStorageFixture
103-
keystoreFile 'gcs.client.integration_test.credentials_file', "${serviceAccountFile.absolutePath}"
104-
105-
/* Use a closure on the string to delay evaluation until tests are executed */
106-
setting 'gcs.client.integration_test.endpoint', "http://${ -> googleCloudStorageFixture.addressAndPort }"
107-
}
56+
check {
57+
// also execute the QA tests when testing the plugin
58+
dependsOn 'qa:google-cloud-storage:check'
59+
}

‎plugins/repository-gcs/qa/build.gradle

Whitespace-only changes.

‎plugins/repository-gcs/qa/google-cloud-storage/build.gradle

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
import org.elasticsearch.gradle.MavenFilteringHack
21+
import org.elasticsearch.gradle.test.AntFixture
22+
23+
import java.security.KeyPair
24+
import java.security.KeyPairGenerator
25+
26+
apply plugin: 'elasticsearch.standalone-rest-test'
27+
apply plugin: 'elasticsearch.rest-test'
28+
29+
dependencies {
30+
testCompile project(path: ':plugins:repository-gcs', configuration: 'runtime')
31+
}
32+
33+
integTestCluster {
34+
plugin ':plugins:repository-gcs'
35+
}
36+
37+
forbiddenApisTest {
38+
// we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
39+
bundledSignatures -= 'jdk-non-portable'
40+
bundledSignatures += 'jdk-internal'
41+
}
42+
43+
boolean useFixture = false
44+
45+
String gcsServiceAccount = System.getenv("google_storage_service_account")
46+
String gcsBucket = System.getenv("google_storage_bucket")
47+
String gcsBasePath = System.getenv("google_storage_base_path")
48+
49+
File serviceAccountFile = null
50+
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
51+
serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
52+
gcsBucket = 'bucket_test'
53+
gcsBasePath = 'integration_test'
54+
useFixture = true
55+
} else {
56+
serviceAccountFile = new File(gcsServiceAccount)
57+
if (serviceAccountFile.exists() == false || serviceAccountFile.canRead() == false) {
58+
throw new FileNotFoundException(gcsServiceAccount, "Google Storage service account file does not exist or is not readable")
59+
}
60+
}
61+
62+
/** A task to start the GoogleCloudStorageFixture which emulates a Google Cloud Storage service **/
63+
task googleCloudStorageFixture(type: AntFixture) {
64+
dependsOn compileTestJava
65+
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
66+
executable = new File(project.runtimeJavaHome, 'bin/java')
67+
args 'org.elasticsearch.repositories.gcs.GoogleCloudStorageFixture', baseDir, 'bucket_test'
68+
}
69+
70+
/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
71+
task createServiceAccountFile() {
72+
dependsOn googleCloudStorageFixture
73+
doLast {
74+
KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA")
75+
keyPairGenerator.initialize(1024)
76+
KeyPair keyPair = keyPairGenerator.generateKeyPair()
77+
String encodedKey = Base64.getEncoder().encodeToString(keyPair.private.getEncoded())
78+
79+
serviceAccountFile.parentFile.mkdirs()
80+
serviceAccountFile.setText("{\n" +
81+
' "type": "service_account",\n' +
82+
' "project_id": "integration_test",\n' +
83+
' "private_key_id": "' + UUID.randomUUID().toString() + '",\n' +
84+
' "private_key": "-----BEGIN PRIVATE KEY-----\\n' + encodedKey + '\\n-----END PRIVATE KEY-----\\n",\n' +
85+
' "client_email": "[email protected]",\n' +
86+
' "client_id": "123456789101112130594",\n' +
87+
" \"auth_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/auth\",\n" +
88+
" \"token_uri\": \"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token\",\n" +
89+
' "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",\n' +
90+
' "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/integration_test%40appspot.gserviceaccount.com"\n' +
91+
'}', 'UTF-8')
92+
}
93+
}
94+
95+
Map<String, Object> expansions = [
96+
'bucket': gcsBucket,
97+
'base_path': gcsBasePath
98+
]
99+
100+
processTestResources {
101+
inputs.properties(expansions)
102+
MavenFilteringHack.filter(it, expansions)
103+
}
104+
105+
integTestCluster {
106+
keystoreFile 'gcs.client.integration_test.credentials_file', "${serviceAccountFile.absolutePath}"
107+
108+
if (useFixture) {
109+
dependsOn createServiceAccountFile, googleCloudStorageFixture
110+
/* Use a closure on the string to delay evaluation until tests are executed */
111+
setting 'gcs.client.integration_test.endpoint', "http://${ -> googleCloudStorageFixture.addressAndPort }"
112+
} else {
113+
println "Using an external service to test the repository-gcs plugin"
114+
}
115+
}

‎plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java renamed to ‎plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java

File renamed without changes.

‎plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepositoryClientYamlTestSuiteIT.java

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.repositories.gcs;
21+
22+
import com.carrotsearch.randomizedtesting.annotations.Name;
23+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
24+
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
25+
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
26+
27+
public class GoogleCloudStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
28+
29+
public GoogleCloudStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
30+
super(testCandidate);
31+
}
32+
33+
@ParametersFactory
34+
public static Iterable<Object[]> parameters() throws Exception {
35+
return createParameters();
36+
}
37+
}

‎plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestServer.java renamed to ‎plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageTestServer.java

File renamed without changes.

‎plugins/repository-gcs/qa/google-cloud-storage/src/test/resources/rest-api-spec/test/repository_gcs/10_repository.yml

+177
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
# Integration tests for repository-gcs
2+
---
3+
"Snapshot/Restore with repository-gcs":
4+
5+
# Register repository
6+
- do:
7+
snapshot.create_repository:
8+
repository: repository
9+
body:
10+
type: gcs
11+
settings:
12+
bucket: ${bucket}
13+
client: "integration_test"
14+
base_path: ${base_path}
15+
16+
- match: { acknowledged: true }
17+
18+
# Get repository
19+
- do:
20+
snapshot.get_repository:
21+
repository: repository
22+
23+
- match: { repository.settings.bucket : ${bucket} }
24+
- match: { repository.settings.client : "integration_test" }
25+
- match: { repository.settings.base_path : ${base_path} }
26+
27+
# Index documents
28+
- do:
29+
bulk:
30+
refresh: true
31+
body:
32+
- index:
33+
_index: docs
34+
_type: doc
35+
_id: 1
36+
- snapshot: one
37+
- index:
38+
_index: docs
39+
_type: doc
40+
_id: 2
41+
- snapshot: one
42+
- index:
43+
_index: docs
44+
_type: doc
45+
_id: 3
46+
- snapshot: one
47+
48+
- do:
49+
count:
50+
index: docs
51+
52+
- match: {count: 3}
53+
54+
# Create a first snapshot
55+
- do:
56+
snapshot.create:
57+
repository: repository
58+
snapshot: snapshot-one
59+
wait_for_completion: true
60+
61+
- match: { snapshot.snapshot: snapshot-one }
62+
- match: { snapshot.state : SUCCESS }
63+
- match: { snapshot.include_global_state: true }
64+
- match: { snapshot.shards.failed : 0 }
65+
66+
- do:
67+
snapshot.status:
68+
repository: repository
69+
snapshot: snapshot-one
70+
71+
- is_true: snapshots
72+
- match: { snapshots.0.snapshot: snapshot-one }
73+
- match: { snapshots.0.state : SUCCESS }
74+
75+
# Index more documents
76+
- do:
77+
bulk:
78+
refresh: true
79+
body:
80+
- index:
81+
_index: docs
82+
_type: doc
83+
_id: 4
84+
- snapshot: two
85+
- index:
86+
_index: docs
87+
_type: doc
88+
_id: 5
89+
- snapshot: two
90+
- index:
91+
_index: docs
92+
_type: doc
93+
_id: 6
94+
- snapshot: two
95+
- index:
96+
_index: docs
97+
_type: doc
98+
_id: 7
99+
- snapshot: two
100+
101+
- do:
102+
count:
103+
index: docs
104+
105+
- match: {count: 7}
106+
107+
# Create a second snapshot
108+
- do:
109+
snapshot.create:
110+
repository: repository
111+
snapshot: snapshot-two
112+
wait_for_completion: true
113+
114+
- match: { snapshot.snapshot: snapshot-two }
115+
- match: { snapshot.state : SUCCESS }
116+
- match: { snapshot.shards.failed : 0 }
117+
118+
- do:
119+
snapshot.get:
120+
repository: repository
121+
snapshot: snapshot-one,snapshot-two
122+
123+
- is_true: snapshots
124+
- match: { snapshots.0.state : SUCCESS }
125+
- match: { snapshots.1.state : SUCCESS }
126+
127+
# Delete the index
128+
- do:
129+
indices.delete:
130+
index: docs
131+
132+
# Restore the second snapshot
133+
- do:
134+
snapshot.restore:
135+
repository: repository
136+
snapshot: snapshot-two
137+
wait_for_completion: true
138+
139+
- do:
140+
count:
141+
index: docs
142+
143+
- match: {count: 7}
144+
145+
# Delete the index again
146+
- do:
147+
indices.delete:
148+
index: docs
149+
150+
# Restore the first snapshot
151+
- do:
152+
snapshot.restore:
153+
repository: repository
154+
snapshot: snapshot-one
155+
wait_for_completion: true
156+
157+
- do:
158+
count:
159+
index: docs
160+
161+
- match: {count: 3}
162+
163+
# Remove the snapshots
164+
- do:
165+
snapshot.delete:
166+
repository: repository
167+
snapshot: snapshot-two
168+
169+
- do:
170+
snapshot.delete:
171+
repository: repository
172+
snapshot: snapshot-one
173+
174+
# Remove our repository
175+
- do:
176+
snapshot.delete_repository:
177+
repository: repository

0 commit comments

Comments
 (0)