Skip to content

Commit 0aad5fd

Browse files
tlrxtlrx
authored
Move repository-azure fixture test to QA project (elastic#30253)
Similarly to what has been done in for the repository-s3 plugin, this pull request moves the fixture test into a dedicated repository-azure/qa/microsoft-azure-storage project. It also exposes some environment variables which allows to execute the integration tests against the real Azure Storage service. When the environment variables are not defined, the integration tests are executed using the fixture added in elastic#29347. Closes elastic#29349
1 parent 66daaaa commit 0aad5fd

File tree

9 files changed

+313
-279
lines changed

9 files changed

+313
-279
lines changed

‎plugins/repository-azure/build.gradle

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
* specific language governing permissions and limitations
1717
* under the License.
1818
*/
19-
import org.elasticsearch.gradle.test.AntFixture
2019

2120
esplugin {
2221
description 'The Azure Repository plugin adds support for Azure storage repositories.'
@@ -43,28 +42,12 @@ thirdPartyAudit.excludes = [
4342
'org.slf4j.LoggerFactory',
4443
]
4544

46-
forbiddenApisTest {
47-
// we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
48-
bundledSignatures -= 'jdk-non-portable'
49-
bundledSignatures += 'jdk-internal'
50-
}
51-
52-
/** A task to start the fixture which emulates an Azure Storage service **/
53-
task azureStorageFixture(type: AntFixture) {
54-
dependsOn compileTestJava
55-
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
56-
executable = new File(project.runtimeJavaHome, 'bin/java')
57-
args 'org.elasticsearch.repositories.azure.AzureStorageFixture', baseDir, 'container_test'
45+
check {
46+
// also execute the QA tests when testing the plugin
47+
dependsOn 'qa:microsoft-azure-storage:check'
5848
}
5949

6050
integTestCluster {
61-
dependsOn azureStorageFixture
62-
63-
keystoreSetting 'azure.client.integration_test.account', "azure_integration_test_account"
64-
/* The key is "azure_integration_test_key" encoded using base64 */
65-
keystoreSetting 'azure.client.integration_test.key', "YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk="
66-
// Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used
67-
// in a hacky way to change the protocol and endpoint. We must fix that.
68-
setting 'azure.client.integration_test.endpoint_suffix',
69-
"ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${ -> azureStorageFixture.addressAndPort }"
70-
}
51+
keystoreSetting 'azure.client.integration_test.account', 'azure_account'
52+
keystoreSetting 'azure.client.integration_test.key', 'azure_key'
53+
}

‎plugins/repository-azure/qa/build.gradle

Whitespace-only changes.

‎plugins/repository-azure/qa/microsoft-azure-storage/build.gradle

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
import org.elasticsearch.gradle.MavenFilteringHack
21+
import org.elasticsearch.gradle.test.AntFixture
22+
23+
apply plugin: 'elasticsearch.standalone-rest-test'
24+
apply plugin: 'elasticsearch.rest-test'
25+
26+
dependencies {
27+
testCompile project(path: ':plugins:repository-azure', configuration: 'runtime')
28+
}
29+
30+
integTestCluster {
31+
plugin ':plugins:repository-azure'
32+
}
33+
34+
forbiddenApisTest {
35+
// we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
36+
bundledSignatures -= 'jdk-non-portable'
37+
bundledSignatures += 'jdk-internal'
38+
}
39+
40+
boolean useFixture = false
41+
42+
String azureAccount = System.getenv("azure_storage_account")
43+
String azureKey = System.getenv("azure_storage_key")
44+
String azureContainer = System.getenv("azure_storage_container")
45+
String azureBasePath = System.getenv("azure_storage_base_path")
46+
47+
if (!azureAccount && !azureKey && !azureContainer && !azureBasePath) {
48+
azureAccount = 'azure_integration_test_account'
49+
azureKey = 'YXp1cmVfaW50ZWdyYXRpb25fdGVzdF9rZXk=' // The key is "azure_integration_test_key" encoded using base64
50+
azureContainer = 'container_test'
51+
azureBasePath = 'integration_test'
52+
useFixture = true
53+
}
54+
55+
/** A task to start the fixture which emulates an Azure Storage service **/
56+
task azureStorageFixture(type: AntFixture) {
57+
dependsOn compileTestJava
58+
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
59+
executable = new File(project.runtimeJavaHome, 'bin/java')
60+
args 'org.elasticsearch.repositories.azure.AzureStorageFixture', baseDir, azureContainer
61+
}
62+
63+
Map<String, Object> expansions = [
64+
'container': azureContainer,
65+
'base_path': azureBasePath
66+
]
67+
processTestResources {
68+
inputs.properties(expansions)
69+
MavenFilteringHack.filter(it, expansions)
70+
}
71+
72+
integTestCluster {
73+
keystoreSetting 'azure.client.integration_test.account', azureAccount
74+
keystoreSetting 'azure.client.integration_test.key', azureKey
75+
76+
if (useFixture) {
77+
dependsOn azureStorageFixture
78+
// Use a closure on the string to delay evaluation until tests are executed. The endpoint_suffix is used
79+
// in a hacky way to change the protocol and endpoint. We must fix that.
80+
setting 'azure.client.integration_test.endpoint_suffix',
81+
"ignored;DefaultEndpointsProtocol=http;BlobEndpoint=http://${ -> azureStorageFixture.addressAndPort }"
82+
} else {
83+
println "Using an external service to test the repository-azure plugin"
84+
}
85+
}

‎plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java renamed to ‎plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java

File renamed without changes.

‎plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageRepositoryClientYamlTestSuiteIT.java

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.repositories.azure;
21+
22+
import com.carrotsearch.randomizedtesting.annotations.Name;
23+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
24+
import org.elasticsearch.common.settings.Settings;
25+
import org.elasticsearch.test.rest.ESRestTestCase;
26+
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
27+
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
28+
29+
public class AzureStorageRepositoryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
30+
31+
public AzureStorageRepositoryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
32+
super(testCandidate);
33+
}
34+
35+
@ParametersFactory
36+
public static Iterable<Object[]> parameters() throws Exception {
37+
return ESClientYamlSuiteTestCase.createParameters();
38+
}
39+
40+
@Override
41+
protected Settings restClientSettings() {
42+
// Give more time to repository-azure to complete the snapshot operations
43+
return Settings.builder().put(super.restClientSettings())
44+
.put(ESRestTestCase.CLIENT_RETRY_TIMEOUT, "60s")
45+
.put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "60s")
46+
.build();
47+
}
48+
}

‎plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureStorageTestServer.java renamed to ‎plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageTestServer.java

File renamed without changes.

‎plugins/repository-azure/qa/microsoft-azure-storage/src/test/resources/rest-api-spec/test/repository_azure/10_repository.yml

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
# Integration tests for repository-azure
2+
---
3+
"Snapshot/Restore with repository-azure":
4+
5+
# Register repository
6+
- do:
7+
snapshot.create_repository:
8+
repository: repository
9+
body:
10+
type: azure
11+
settings:
12+
container: ${container}
13+
client: "integration_test"
14+
base_path: ${base_path}
15+
16+
- match: { acknowledged: true }
17+
18+
# Get repository
19+
- do:
20+
snapshot.get_repository:
21+
repository: repository
22+
23+
- match: { repository.settings.container: ${container} }
24+
- match: { repository.settings.client : "integration_test" }
25+
- match: { repository.settings.base_path : ${base_path} }
26+
27+
# Index documents
28+
- do:
29+
bulk:
30+
refresh: true
31+
body:
32+
- index:
33+
_index: docs
34+
_type: doc
35+
_id: 1
36+
- snapshot: one
37+
- index:
38+
_index: docs
39+
_type: doc
40+
_id: 2
41+
- snapshot: one
42+
- index:
43+
_index: docs
44+
_type: doc
45+
_id: 3
46+
- snapshot: one
47+
48+
- do:
49+
count:
50+
index: docs
51+
52+
- match: {count: 3}
53+
54+
# Create a first snapshot
55+
- do:
56+
snapshot.create:
57+
repository: repository
58+
snapshot: snapshot-one
59+
wait_for_completion: true
60+
61+
- match: { snapshot.snapshot: snapshot-one }
62+
- match: { snapshot.state : SUCCESS }
63+
- match: { snapshot.include_global_state: true }
64+
- match: { snapshot.shards.failed : 0 }
65+
66+
- do:
67+
snapshot.status:
68+
repository: repository
69+
snapshot: snapshot-one
70+
71+
- is_true: snapshots
72+
- match: { snapshots.0.snapshot: snapshot-one }
73+
- match: { snapshots.0.state : SUCCESS }
74+
75+
# Index more documents
76+
- do:
77+
bulk:
78+
refresh: true
79+
body:
80+
- index:
81+
_index: docs
82+
_type: doc
83+
_id: 4
84+
- snapshot: two
85+
- index:
86+
_index: docs
87+
_type: doc
88+
_id: 5
89+
- snapshot: two
90+
- index:
91+
_index: docs
92+
_type: doc
93+
_id: 6
94+
- snapshot: two
95+
- index:
96+
_index: docs
97+
_type: doc
98+
_id: 7
99+
- snapshot: two
100+
101+
- do:
102+
count:
103+
index: docs
104+
105+
- match: {count: 7}
106+
107+
# Create a second snapshot
108+
- do:
109+
snapshot.create:
110+
repository: repository
111+
snapshot: snapshot-two
112+
wait_for_completion: true
113+
114+
- match: { snapshot.snapshot: snapshot-two }
115+
- match: { snapshot.state : SUCCESS }
116+
- match: { snapshot.shards.failed : 0 }
117+
118+
- do:
119+
snapshot.get:
120+
repository: repository
121+
snapshot: snapshot-one,snapshot-two
122+
123+
- is_true: snapshots
124+
- match: { snapshots.0.state : SUCCESS }
125+
- match: { snapshots.1.state : SUCCESS }
126+
127+
# Delete the index
128+
- do:
129+
indices.delete:
130+
index: docs
131+
132+
# Restore the second snapshot
133+
- do:
134+
snapshot.restore:
135+
repository: repository
136+
snapshot: snapshot-two
137+
wait_for_completion: true
138+
139+
- do:
140+
count:
141+
index: docs
142+
143+
- match: {count: 7}
144+
145+
# Delete the index again
146+
- do:
147+
indices.delete:
148+
index: docs
149+
150+
# Restore the first snapshot
151+
- do:
152+
snapshot.restore:
153+
repository: repository
154+
snapshot: snapshot-one
155+
wait_for_completion: true
156+
157+
- do:
158+
count:
159+
index: docs
160+
161+
- match: {count: 3}
162+
163+
# Remove the snapshots
164+
- do:
165+
snapshot.delete:
166+
repository: repository
167+
snapshot: snapshot-two
168+
master_timeout: 5m
169+
170+
- do:
171+
snapshot.delete:
172+
repository: repository
173+
snapshot: snapshot-one
174+
master_timeout: 5m

0 commit comments

Comments
 (0)