Skip to content

Commit 5b115fb

Browse files
authored
Add S3 integration tests for searchable snapshots (#52263)
This commit adds REST-based integration tests that test the searchable snapshots feature on S3.
1 parent 957a7ae commit 5b115fb

File tree

10 files changed

+410
-10
lines changed

10 files changed

+410
-10
lines changed

plugins/repository-s3/build.gradle

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,12 @@ processTestResources {
214214
MavenFilteringHack.filter(it, expansions)
215215
}
216216

217-
testFixtures.useFixture(':test:fixtures:s3-fixture')
217+
[
218+
's3-fixture',
219+
's3-fixture-with-session-token',
220+
's3-fixture-with-ec2',
221+
's3-fixture-with-ecs',
222+
].forEach { fixture -> testFixtures.useFixture(':test:fixtures:s3-fixture', fixture) }
218223

219224
def fixtureAddress = { fixture ->
220225
assert useFixture: 'closure should not be used without a fixture'

test/fixtures/s3-fixture/docker-compose.yml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,21 @@ services:
1515
ports:
1616
- "80"
1717

18+
s3-fixture-other:
19+
build:
20+
context: .
21+
args:
22+
fixtureClass: fixture.s3.S3HttpFixture
23+
port: 80
24+
bucket: "bucket"
25+
basePath: "base_path"
26+
accessKey: "access_key"
27+
dockerfile: Dockerfile
28+
volumes:
29+
- ./testfixtures_shared/shared:/fixture/shared
30+
ports:
31+
- "80"
32+
1833
s3-fixture-with-session-token:
1934
build:
2035
context: .

test/fixtures/s3-fixture/src/main/java/fixture/s3/S3HttpHandler.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -216,13 +216,13 @@ public void handle(final HttpExchange exchange) throws IOException {
216216

217217
final int start = Integer.parseInt(matcher.group(1));
218218
final int end = Integer.parseInt(matcher.group(2));
219-
final int length = end - start;
220219

220+
final BytesReference rangeBlob = blob.slice(start, end + 1 - start);
221221
exchange.getResponseHeaders().add("Content-Type", "application/octet-stream");
222-
exchange.getResponseHeaders().add("Content-Range",
223-
String.format(Locale.ROOT, "bytes=%d-%d/%d", start, end, blob.length()));
224-
exchange.sendResponseHeaders(RestStatus.OK.getStatus(), length);
225-
exchange.getResponseBody().write(BytesReference.toBytes(blob), start, length);
222+
exchange.getResponseHeaders().add("Content-Range", String.format(Locale.ROOT, "bytes %d-%d/%d",
223+
start, end, rangeBlob.length()));
224+
exchange.sendResponseHeaders(RestStatus.OK.getStatus(), rangeBlob.length());
225+
rangeBlob.writeTo(exchange.getResponseBody());
226226
}
227227
} else {
228228
exchange.sendResponseHeaders(RestStatus.NOT_FOUND.getStatus(), -1);

x-pack/plugin/searchable-snapshots/build.gradle

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,3 +27,16 @@ gradle.projectsEvaluated {
2727
.findAll { it.path.startsWith(project.path + ":qa") }
2828
.each { check.dependsOn it.check }
2929
}
30+
31+
configurations {
32+
testArtifacts.extendsFrom testRuntime
33+
}
34+
35+
task testJar(type: Jar) {
36+
appendix 'test'
37+
from sourceSets.test.output
38+
}
39+
40+
artifacts {
41+
testArtifacts testJar
42+
}

x-pack/plugin/searchable-snapshots/qa/rest/build.gradle

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,17 @@ apply plugin: 'elasticsearch.testclusters'
22
apply plugin: 'elasticsearch.standalone-rest-test'
33
apply plugin: 'elasticsearch.rest-test'
44

5+
dependencies {
6+
testCompile project(path: xpackModule('searchable-snapshots'), configuration: 'testArtifacts')
7+
}
8+
9+
final File repoDir = file("$buildDir/testclusters/repo")
10+
11+
integTest.runner {
12+
systemProperty 'tests.path.repo', repoDir
13+
}
14+
515
testClusters.integTest {
616
testDistribution = 'DEFAULT'
7-
setting 'xpack.license.self_generated.type', 'basic'
17+
setting 'path.repo', repoDir.absolutePath
818
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.searchablesnapshots.rest;
7+
8+
import org.elasticsearch.common.settings.Settings;
9+
import org.elasticsearch.common.unit.ByteSizeUnit;
10+
import org.elasticsearch.repositories.fs.FsRepository;
11+
import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsRestTestCase;
12+
13+
public class FsSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
14+
15+
@Override
16+
protected String repositoryType() {
17+
return FsRepository.TYPE;
18+
}
19+
20+
@Override
21+
protected Settings repositorySettings() {
22+
final Settings.Builder settings = Settings.builder();
23+
settings.put("location", System.getProperty("tests.path.repo"));
24+
if (randomBoolean()) {
25+
settings.put("compress", randomBoolean());
26+
}
27+
if (randomBoolean()) {
28+
settings.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES);
29+
}
30+
return settings.build();
31+
}
32+
}
Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,15 @@
33
* or more contributor license agreements. Licensed under the Elastic License;
44
* you may not use this file except in compliance with the Elastic License.
55
*/
6-
76
package org.elasticsearch.xpack.searchablesnapshots.rest;
87

98
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
109
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
1110
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
1211

13-
public class SearchableSnapshotsRestIT extends ESClientYamlSuiteTestCase {
12+
public class SearchableSnapshotsClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
1413

15-
public SearchableSnapshotsRestIT(final ClientYamlTestCandidate testCandidate) {
14+
public SearchableSnapshotsClientYamlTestSuiteIT(final ClientYamlTestCandidate testCandidate) {
1615
super(testCandidate);
1716
}
1817

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
2+
3+
apply plugin: 'elasticsearch.standalone-rest-test'
4+
apply plugin: 'elasticsearch.rest-test'
5+
6+
final Project fixture = project(':test:fixtures:s3-fixture')
7+
final Project repositoryPlugin = project(':plugins:repository-s3')
8+
9+
dependencies {
10+
testCompile project(path: xpackModule('searchable-snapshots'), configuration: 'testArtifacts')
11+
testCompile repositoryPlugin
12+
}
13+
14+
boolean useFixture = false
15+
String s3AccessKey = System.getenv("amazon_s3_access_key")
16+
String s3SecretKey = System.getenv("amazon_s3_secret_key")
17+
String s3Bucket = System.getenv("amazon_s3_bucket")
18+
String s3BasePath = System.getenv("amazon_s3_base_path")
19+
20+
if (!s3AccessKey && !s3SecretKey && !s3Bucket && !s3BasePath) {
21+
s3AccessKey = 'access_key'
22+
s3SecretKey = 'secret_key'
23+
s3Bucket = 'bucket'
24+
s3BasePath = 'base_path'
25+
useFixture = true
26+
27+
} else if (!s3AccessKey || !s3SecretKey || !s3Bucket || !s3BasePath) {
28+
throw new IllegalArgumentException("not all options specified to run against external S3 service are present")
29+
}
30+
31+
if (useFixture) {
32+
apply plugin: 'elasticsearch.test.fixtures'
33+
testFixtures.useFixture(fixture.path, 's3-fixture-other')
34+
}
35+
36+
integTest {
37+
dependsOn repositoryPlugin.bundlePlugin
38+
runner {
39+
systemProperty 'test.s3.bucket', s3Bucket
40+
systemProperty 'test.s3.base_path', s3BasePath + "/searchable_snapshots_tests"
41+
}
42+
}
43+
44+
testClusters.integTest {
45+
testDistribution = 'DEFAULT'
46+
plugin file(repositoryPlugin.bundlePlugin.archiveFile)
47+
48+
keystore 's3.client.searchable_snapshots.access_key', s3AccessKey
49+
keystore 's3.client.searchable_snapshots.secret_key', s3SecretKey
50+
51+
if (useFixture) {
52+
def fixtureAddress = { fixtureName ->
53+
assert useFixture: 'closure should not be used without a fixture'
54+
int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80"
55+
assert ephemeralPort > 0
56+
'127.0.0.1:' + ephemeralPort
57+
}
58+
59+
setting 's3.client.searchable_snapshots.protocol', 'http'
60+
setting 's3.client.searchable_snapshots.endpoint', { "${-> fixtureAddress('s3-fixture-other')}" }, IGNORE_VALUE
61+
62+
} else {
63+
println "Using an external service to test " + project.name
64+
}
65+
}
66+
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License;
4+
* you may not use this file except in compliance with the Elastic License.
5+
*/
6+
package org.elasticsearch.xpack.searchablesnapshots.s3;
7+
8+
import org.elasticsearch.common.settings.Settings;
9+
import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsRestTestCase;
10+
11+
import static org.hamcrest.Matchers.blankOrNullString;
12+
import static org.hamcrest.Matchers.not;
13+
14+
public class S3SearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
15+
16+
@Override
17+
protected String repositoryType() {
18+
return "s3";
19+
}
20+
21+
@Override
22+
protected Settings repositorySettings() {
23+
final String bucket = System.getProperty("test.s3.bucket");
24+
assertThat(bucket, not(blankOrNullString()));
25+
26+
final String basePath = System.getProperty("test.s3.base_path");
27+
assertThat(basePath, not(blankOrNullString()));
28+
29+
return Settings.builder()
30+
.put("client", "searchable_snapshots")
31+
.put("bucket", bucket)
32+
.put("base_path", basePath)
33+
.build();
34+
}
35+
}

0 commit comments

Comments
 (0)