Skip to content

Commit 7ae3b3b

Browse files
tlrxtlrx
authored
Move repository-s3 fixture tests to QA test project (#29372)
This commit moves the repository-s3 fixture test added in #29296 in a new `repository-s3/qa/amazon-s3` project. This new project allows the REST integration tests to be executed using the real S3 service when all the required environment variables are provided. When no env var is provided, then the tests are executed using the fixture added in #29296. The REST tests located at the `repository-s3`plugin project now only verify that the plugin is correctly loaded. The REST tests have been adapted to allow a bucket name and a base path to be specified as env vars. This way it is possible to run the tests with different base paths (could be anything, like a CI job name or a branch name) without multiplicating buckets. Related to #29349
1 parent 63148dd commit 7ae3b3b

File tree

13 files changed

+318
-207
lines changed

13 files changed

+318
-207
lines changed

‎plugins/repository-s3/build.gradle

+3-19
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
import org.elasticsearch.gradle.test.AntFixture
2-
31
/*
42
* Licensed to Elasticsearch under one or more contributor
53
* license agreements. See the NOTICE file distributed with
@@ -66,28 +64,14 @@ test {
6664
exclude '**/*CredentialsTests.class'
6765
}
6866

69-
forbiddenApisTest {
70-
// we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
71-
bundledSignatures -= 'jdk-non-portable'
72-
bundledSignatures += 'jdk-internal'
73-
}
74-
75-
/** A task to start the AmazonS3Fixture which emulates a S3 service **/
76-
task s3Fixture(type: AntFixture) {
77-
dependsOn compileTestJava
78-
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
79-
executable = new File(project.runtimeJavaHome, 'bin/java')
80-
args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, 'bucket_test'
67+
check {
68+
// also execute the QA tests when testing the plugin
69+
dependsOn 'qa:amazon-s3:check'
8170
}
8271

8372
integTestCluster {
84-
dependsOn s3Fixture
85-
8673
keystoreSetting 's3.client.integration_test.access_key', "s3_integration_test_access_key"
8774
keystoreSetting 's3.client.integration_test.secret_key', "s3_integration_test_secret_key"
88-
89-
/* Use a closure on the string to delay evaluation until tests are executed */
90-
setting 's3.client.integration_test.endpoint', "http://${ -> s3Fixture.addressAndPort }"
9175
}
9276

9377
thirdPartyAudit.excludes = [

‎plugins/repository-s3/qa/amazon-s3/build.gradle

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
import org.elasticsearch.gradle.MavenFilteringHack
21+
import org.elasticsearch.gradle.test.AntFixture
22+
23+
apply plugin: 'elasticsearch.standalone-rest-test'
24+
apply plugin: 'elasticsearch.rest-test'
25+
26+
dependencies {
27+
testCompile project(path: ':plugins:repository-s3', configuration: 'runtime')
28+
}
29+
30+
integTestCluster {
31+
plugin ':plugins:repository-s3'
32+
}
33+
34+
forbiddenApisTest {
35+
// we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
36+
bundledSignatures -= 'jdk-non-portable'
37+
bundledSignatures += 'jdk-internal'
38+
}
39+
40+
boolean useFixture = false
41+
42+
String s3AccessKey = System.getenv("amazon_s3_access_key")
43+
String s3SecretKey = System.getenv("amazon_s3_secret_key")
44+
String s3Bucket = System.getenv("amazon_s3_bucket")
45+
String s3BasePath = System.getenv("amazon_s3_base_path")
46+
47+
if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
48+
s3AccessKey = 's3_integration_test_access_key'
49+
s3SecretKey = 's3_integration_test_secret_key'
50+
s3Bucket = 'bucket_test'
51+
s3BasePath = 'integration_test'
52+
useFixture = true
53+
}
54+
55+
/** A task to start the AmazonS3Fixture which emulates a S3 service **/
56+
task s3Fixture(type: AntFixture) {
57+
dependsOn compileTestJava
58+
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
59+
executable = new File(project.runtimeJavaHome, 'bin/java')
60+
args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3Bucket
61+
}
62+
63+
Map<String, Object> expansions = [
64+
'bucket': s3Bucket,
65+
'base_path': s3BasePath
66+
]
67+
processTestResources {
68+
inputs.properties(expansions)
69+
MavenFilteringHack.filter(it, expansions)
70+
}
71+
72+
integTestCluster {
73+
keystoreSetting 's3.client.integration_test.access_key', s3AccessKey
74+
keystoreSetting 's3.client.integration_test.secret_key', s3SecretKey
75+
76+
if (useFixture) {
77+
dependsOn s3Fixture
78+
/* Use a closure on the string to delay evaluation until tests are executed */
79+
setting 's3.client.integration_test.endpoint', "http://${-> s3Fixture.addressAndPort}"
80+
} else {
81+
println "Using an external service to test the repository-s3 plugin"
82+
}
83+
}

‎plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java renamed to ‎plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java

File renamed without changes.

‎plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3RepositoryClientYamlTestSuiteIT.java

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.repositories.s3;
21+
22+
import com.carrotsearch.randomizedtesting.annotations.Name;
23+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
24+
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
25+
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
26+
27+
public class AmazonS3RepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
28+
29+
public AmazonS3RepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
30+
super(testCandidate);
31+
}
32+
33+
@ParametersFactory
34+
public static Iterable<Object[]> parameters() throws Exception {
35+
return ESClientYamlSuiteTestCase.createParameters();
36+
}
37+
}

‎plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java renamed to ‎plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3TestServer.java

File renamed without changes.

‎plugins/repository-s3/qa/amazon-s3/src/test/resources/rest-api-spec/test/repository_s3/10_repository.yml

+183
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
# Integration tests for repository-s3
2+
---
3+
"Snapshot/Restore with repository-s3":
4+
5+
# Register repository
6+
- do:
7+
snapshot.create_repository:
8+
repository: repository
9+
body:
10+
type: s3
11+
settings:
12+
bucket: ${bucket}
13+
client: integration_test
14+
base_path: ${base_path}
15+
canned_acl: private
16+
storage_class: standard
17+
18+
- match: { acknowledged: true }
19+
20+
# Get repository
21+
- do:
22+
snapshot.get_repository:
23+
repository: repository
24+
25+
- match: { repository.settings.bucket : ${bucket} }
26+
- match: { repository.settings.client : "integration_test" }
27+
- match: { repository.settings.base_path : ${base_path} }
28+
- match: { repository.settings.canned_acl : "private" }
29+
- match: { repository.settings.storage_class : "standard" }
30+
- is_false: repository.settings.access_key
31+
- is_false: repository.settings.secret_key
32+
33+
# Index documents
34+
- do:
35+
bulk:
36+
refresh: true
37+
body:
38+
- index:
39+
_index: docs
40+
_type: doc
41+
_id: 1
42+
- snapshot: one
43+
- index:
44+
_index: docs
45+
_type: doc
46+
_id: 2
47+
- snapshot: one
48+
- index:
49+
_index: docs
50+
_type: doc
51+
_id: 3
52+
- snapshot: one
53+
54+
- do:
55+
count:
56+
index: docs
57+
58+
- match: {count: 3}
59+
60+
# Create a first snapshot
61+
- do:
62+
snapshot.create:
63+
repository: repository
64+
snapshot: snapshot-one
65+
wait_for_completion: true
66+
67+
- match: { snapshot.snapshot: snapshot-one }
68+
- match: { snapshot.state : SUCCESS }
69+
- match: { snapshot.include_global_state: true }
70+
- match: { snapshot.shards.failed : 0 }
71+
72+
- do:
73+
snapshot.status:
74+
repository: repository
75+
snapshot: snapshot-one
76+
77+
- is_true: snapshots
78+
- match: { snapshots.0.snapshot: snapshot-one }
79+
- match: { snapshots.0.state : SUCCESS }
80+
81+
# Index more documents
82+
- do:
83+
bulk:
84+
refresh: true
85+
body:
86+
- index:
87+
_index: docs
88+
_type: doc
89+
_id: 4
90+
- snapshot: two
91+
- index:
92+
_index: docs
93+
_type: doc
94+
_id: 5
95+
- snapshot: two
96+
- index:
97+
_index: docs
98+
_type: doc
99+
_id: 6
100+
- snapshot: two
101+
- index:
102+
_index: docs
103+
_type: doc
104+
_id: 7
105+
- snapshot: two
106+
107+
- do:
108+
count:
109+
index: docs
110+
111+
- match: {count: 7}
112+
113+
# Create a second snapshot
114+
- do:
115+
snapshot.create:
116+
repository: repository
117+
snapshot: snapshot-two
118+
wait_for_completion: true
119+
120+
- match: { snapshot.snapshot: snapshot-two }
121+
- match: { snapshot.state : SUCCESS }
122+
- match: { snapshot.shards.failed : 0 }
123+
124+
- do:
125+
snapshot.get:
126+
repository: repository
127+
snapshot: snapshot-one,snapshot-two
128+
129+
- is_true: snapshots
130+
- match: { snapshots.0.state : SUCCESS }
131+
- match: { snapshots.1.state : SUCCESS }
132+
133+
# Delete the index
134+
- do:
135+
indices.delete:
136+
index: docs
137+
138+
# Restore the second snapshot
139+
- do:
140+
snapshot.restore:
141+
repository: repository
142+
snapshot: snapshot-two
143+
wait_for_completion: true
144+
145+
- do:
146+
count:
147+
index: docs
148+
149+
- match: {count: 7}
150+
151+
# Delete the index again
152+
- do:
153+
indices.delete:
154+
index: docs
155+
156+
# Restore the first snapshot
157+
- do:
158+
snapshot.restore:
159+
repository: repository
160+
snapshot: snapshot-one
161+
wait_for_completion: true
162+
163+
- do:
164+
count:
165+
index: docs
166+
167+
- match: {count: 3}
168+
169+
# Remove the snapshots
170+
- do:
171+
snapshot.delete:
172+
repository: repository
173+
snapshot: snapshot-two
174+
175+
- do:
176+
snapshot.delete:
177+
repository: repository
178+
snapshot: snapshot-one
179+
180+
# Remove our repository
181+
- do:
182+
snapshot.delete_repository:
183+
repository: repository

‎plugins/repository-s3/qa/build.gradle

Whitespace-only changes.

‎plugins/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Repository.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ class S3Repository extends BlobStoreRepository {
156156

157157
String bucket = BUCKET_SETTING.get(metadata.settings());
158158
if (bucket == null) {
159-
throw new RepositoryException(metadata.name(), "No bucket defined for s3 gateway");
159+
throw new RepositoryException(metadata.name(), "No bucket defined for s3 repository");
160160
}
161161

162162
boolean serverSideEncryption = SERVER_SIDE_ENCRYPTION_SETTING.get(metadata.settings());

0 commit comments

Comments
 (0)