Skip to content

Commit 3f8a925

Browse files
fcofdezfcofdez
authored
Add searchable snapshots integration tests for URL repositories (#70709)
Relates #69521
1 parent 35af0bb commit 3f8a925

File tree

22 files changed

+298
-101
lines changed

22 files changed

+298
-101
lines changed

‎modules/repository-url/build.gradle

Lines changed: 8 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@
77
*/
88

99
import org.elasticsearch.gradle.PropertyNormalization
10-
import org.elasticsearch.gradle.info.BuildParams
11-
import org.elasticsearch.gradle.test.AntFixture
1210

1311
apply plugin: 'elasticsearch.yaml-rest-test'
1412
apply plugin: 'elasticsearch.internal-cluster-test'
1513
apply plugin: 'elasticsearch.test.fixtures'
1614

15+
final Project fixture = project(':test:fixtures:url-fixture')
16+
1717
esplugin {
1818
description 'Module for URL repository'
1919
classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
@@ -43,41 +43,22 @@ tasks.named("thirdPartyAudit").configure {
4343
)
4444
}
4545

46-
// This directory is shared between two URL repositories and one FS repository in YAML integration tests
47-
File repositoryDir = new File(project.buildDir, "shared-repository")
48-
49-
def testJar = tasks.register("testJar", Jar) {
50-
from sourceSets.test.output
51-
classifier = "test"
52-
}
53-
54-
tasks.named("preProcessFixture").configure {
55-
inputs.files(testJar, sourceSets.test.runtimeClasspath)
56-
doLast {
57-
file("${testFixturesDir}/shared").mkdirs()
58-
project.copy {
59-
from testJar
60-
from sourceSets.test.runtimeClasspath
61-
into "${testFixturesDir}/shared"
62-
}
63-
repositoryDir.mkdirs()
64-
}
65-
}
46+
testFixtures.useFixture(fixture.path, 'url-fixture')
6647

67-
testFixtures.useFixture()
68-
69-
def fixtureAddress = {
70-
int ephemeralPort = tasks.named("postProcessFixture").get().ext."test.fixtures.url-fixture.tcp.80"
48+
def fixtureAddress = { fixtureName ->
49+
int ephemeralPort = fixture.postProcessFixture.ext."test.fixtures.${fixtureName}.tcp.80"
7150
assert ephemeralPort > 0
7251
'http://127.0.0.1:' + ephemeralPort
7352
}
7453

54+
File repositoryDir = fixture.fsRepositoryDir as File
55+
7556
testClusters.all {
7657
// repositoryDir is used by a FS repository to create snapshots
7758
setting 'path.repo', "${repositoryDir.absolutePath}", PropertyNormalization.IGNORE_VALUE
7859
// repositoryDir is used by two URL repositories to restore snapshots
7960
setting 'repositories.url.allowed_urls', {
80-
"http://snapshot.test*,${fixtureAddress()}"
61+
"http://snapshot.test*,${fixtureAddress('url-fixture')}"
8162
}, PropertyNormalization.IGNORE_VALUE
8263
}
8364

‎modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/http/URLHttpClient.java

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,20 @@
3636

3737
public class URLHttpClient implements Closeable {
3838
public static final int MAX_ERROR_MESSAGE_BODY_SIZE = 1024;
39+
private static final int MAX_CONNECTIONS = 50;
3940
private final Logger logger = LogManager.getLogger(URLHttpClient.class);
4041

4142
private final CloseableHttpClient client;
4243
private final URLHttpClientSettings httpClientSettings;
4344

4445
public static class Factory implements Closeable {
45-
private final PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager();
46+
private final PoolingHttpClientConnectionManager connManager;
47+
48+
public Factory() {
49+
this.connManager = new PoolingHttpClientConnectionManager();
50+
connManager.setDefaultMaxPerRoute(MAX_CONNECTIONS);
51+
connManager.setMaxTotal(MAX_CONNECTIONS);
52+
}
4653

4754
public URLHttpClient create(URLHttpClientSettings settings) {
4855
final CloseableHttpClient apacheHttpClient = HttpClients.custom()

‎settings.gradle

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ List projects = [
6060
'test:fixtures:old-elasticsearch',
6161
'test:fixtures:s3-fixture',
6262
'test:fixtures:geoip-fixture',
63+
'test:fixtures:url-fixture',
64+
'test:fixtures:nginx-fixture',
6365
'test:logger-usage'
6466
]
6567

‎test/fixtures/nginx-fixture/Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
FROM nginx
2+
COPY nginx.conf /etc/nginx/nginx.conf

‎test/fixtures/nginx-fixture/build.gradle

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
apply plugin: 'elasticsearch.test.fixtures'
9+
10+
description = 'Fixture for an external http service'
11+
12+
// These directories are shared between the URL repository and the FS repository in integration tests
13+
project.ext {
14+
fsRepositoryDir = file("${testFixturesDir}/fs-repository")
15+
}
16+
17+
tasks.named("preProcessFixture").configure {
18+
doLast {
19+
// tests expect to have an empty repo
20+
project.ext.fsRepositoryDir.mkdirs()
21+
}
22+
}

‎test/fixtures/nginx-fixture/docker-compose.yml

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
version: '3'
2+
services:
3+
nginx-fixture:
4+
build:
5+
context: .
6+
volumes:
7+
- ./testfixtures_shared/fs-repository:/data
8+
ports:
9+
- "80"

‎test/fixtures/nginx-fixture/nginx.conf

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
events {}
2+
3+
http {
4+
server {
5+
listen 80 default_server;
6+
listen [::]:80 default_server;
7+
8+
root /data;
9+
}
10+
}

‎modules/repository-url/Dockerfile renamed to ‎test/fixtures/url-fixture/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,6 @@ ENV URL_FIXTURE_WORKING_DIR=${workingDir}
1212
ENV URL_FIXTURE_REPO_DIR=${repositoryDir}
1313

1414
ENTRYPOINT exec java -classpath "/fixture/shared/*" \
15-
org.elasticsearch.repositories.url.URLFixture "$URL_FIXTURE_PORT" "$URL_FIXTURE_WORKING_DIR" "$URL_FIXTURE_REPO_DIR"
15+
fixture.url.URLFixture "$URL_FIXTURE_PORT" "$URL_FIXTURE_WORKING_DIR" "$URL_FIXTURE_REPO_DIR"
1616

17-
EXPOSE $port
17+
EXPOSE $port

‎test/fixtures/url-fixture/build.gradle

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0 and the Server Side Public License, v 1; you may not use this file except
5+
* in compliance with, at your election, the Elastic License 2.0 or the Server
6+
* Side Public License, v 1.
7+
*/
8+
apply plugin: 'elasticsearch.java'
9+
apply plugin: 'elasticsearch.test.fixtures'
10+
11+
description = 'Fixture for URL external service'
12+
tasks.named("test").configure { enabled = false }
13+
14+
dependencies {
15+
api project(':server')
16+
api project(':test:framework')
17+
}
18+
19+
// These directories are shared between the URL repository and the FS repository in integration tests
20+
project.ext {
21+
fsRepositoryDir = file("${testFixturesDir}/fs-repository")
22+
}
23+
24+
tasks.named("preProcessFixture").configure {
25+
dependsOn "jar", configurations.runtimeClasspath
26+
doLast {
27+
file("${testFixturesDir}/shared").mkdirs()
28+
project.copy {
29+
from jar
30+
from configurations.runtimeClasspath
31+
into "${testFixturesDir}/shared"
32+
}
33+
project.fsRepositoryDir.mkdirs()
34+
}
35+
}

‎modules/repository-url/docker-compose.yml renamed to ‎test/fixtures/url-fixture/docker-compose.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ services:
99
repositoryDir: "/fixture/repo"
1010
volumes:
1111
- ./testfixtures_shared/shared:/fixture/shared
12-
- ./build/shared-repository:/fixture/repo
12+
- ./testfixtures_shared/fs-repository:/fixture/repo
1313
- ./testfixtures_shared/work:/fixture/work
1414
ports:
15-
- "80"
15+
- "80"

‎modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java renamed to ‎test/fixtures/url-fixture/src/main/java/fixture/url/URLFixture.java

Lines changed: 43 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
* in compliance with, at your election, the Elastic License 2.0 or the Server
66
* Side Public License, v 1.
77
*/
8-
package org.elasticsearch.repositories.url;
8+
package fixture.url;
99

1010
import org.elasticsearch.test.fixture.AbstractHttpFixture;
1111
import org.elasticsearch.common.SuppressForbidden;
@@ -18,13 +18,15 @@
1818
import java.nio.file.Paths;
1919
import java.util.HashMap;
2020
import java.util.Map;
21+
import java.util.regex.Matcher;
22+
import java.util.regex.Pattern;
2123

2224
/**
2325
* This {@link URLFixture} exposes a filesystem directory over HTTP. It is used in repository-url
2426
* integration tests to expose a directory created by a regular FS repository.
2527
*/
2628
public class URLFixture extends AbstractHttpFixture {
27-
29+
private static final Pattern RANGE_PATTERN = Pattern.compile("bytes=(\\d+)-(\\d+)$");
2830
private final Path repositoryDir;
2931

3032
/**
@@ -54,28 +56,55 @@ public static void main(String[] args) throws Exception {
5456
@Override
5557
protected AbstractHttpFixture.Response handle(final Request request) throws IOException {
5658
if ("GET".equalsIgnoreCase(request.getMethod())) {
57-
String path = request.getPath();
58-
if (path.length() > 0 && path.charAt(0) == '/') {
59-
path = path.substring(1);
60-
}
59+
return handleGetRequest(request);
60+
}
61+
return null;
62+
}
6163

62-
Path normalizedRepositoryDir = repositoryDir.normalize();
63-
Path normalizedPath = normalizedRepositoryDir.resolve(path).normalize();
64+
private AbstractHttpFixture.Response handleGetRequest(Request request) throws IOException {
65+
String path = request.getPath();
66+
if (path.length() > 0 && path.charAt(0) == '/') {
67+
path = path.substring(1);
68+
}
69+
70+
Path normalizedRepositoryDir = repositoryDir.normalize();
71+
Path normalizedPath = normalizedRepositoryDir.resolve(path).normalize();
6472

65-
if (normalizedPath.startsWith(normalizedRepositoryDir)) {
66-
if (Files.exists(normalizedPath) && Files.isReadable(normalizedPath) && Files.isRegularFile(normalizedPath)) {
73+
if (normalizedPath.startsWith(normalizedRepositoryDir)) {
74+
if (Files.exists(normalizedPath) && Files.isReadable(normalizedPath) && Files.isRegularFile(normalizedPath)) {
75+
final String range = request.getHeader("Range");
76+
final Map<String, String> headers = new HashMap<>(contentType("application/octet-stream"));
77+
if (range == null) {
6778
byte[] content = Files.readAllBytes(normalizedPath);
68-
final Map<String, String> headers = new HashMap<>(contentType("application/octet-stream"));
6979
headers.put("Content-Length", String.valueOf(content.length));
7080
return new Response(RestStatus.OK.getStatus(), headers, content);
7181
} else {
72-
return new Response(RestStatus.NOT_FOUND.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
82+
final Matcher matcher = RANGE_PATTERN.matcher(range);
83+
if (matcher.matches() == false) {
84+
return new Response(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
85+
} else {
86+
long start = Long.parseLong(matcher.group(1));
87+
long end = Long.parseLong(matcher.group(2));
88+
long rangeLength = end - start + 1;
89+
final long fileSize = Files.size(normalizedPath);
90+
if (start >= fileSize || start > end || rangeLength > fileSize) {
91+
return new Response(RestStatus.REQUESTED_RANGE_NOT_SATISFIED.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
92+
}
93+
94+
headers.put("Content-Length", String.valueOf(rangeLength));
95+
headers.put("Content-Range", "bytes " + start + "-" + end + "/" + fileSize);
96+
final byte[] content = Files.readAllBytes(normalizedPath);
97+
final byte[] responseData = new byte[(int) rangeLength];
98+
System.arraycopy(content, (int) start, responseData, 0, (int) rangeLength);
99+
return new Response(RestStatus.PARTIAL_CONTENT.getStatus(), headers, responseData);
100+
}
73101
}
74102
} else {
75-
return new Response(RestStatus.FORBIDDEN.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
103+
return new Response(RestStatus.NOT_FOUND.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
76104
}
105+
} else {
106+
return new Response(RestStatus.FORBIDDEN.getStatus(), TEXT_PLAIN_CONTENT_TYPE, EMPTY_BYTE);
77107
}
78-
return null;
79108
}
80109

81110
@SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")

‎x-pack/plugin/searchable-snapshots/qa/azure/src/test/java/org/elasticsearch/xpack/searchablesnapshots/AzureSearchableSnapshotsIT.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@
1515
public class AzureSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
1616

1717
@Override
18-
protected String repositoryType() {
18+
protected String writeRepositoryType() {
1919
return "azure";
2020
}
2121

2222
@Override
23-
protected Settings repositorySettings() {
23+
protected Settings writeRepositorySettings() {
2424
final String container = System.getProperty("test.azure.container");
2525
assertThat(container, not(blankOrNullString()));
2626

‎x-pack/plugin/searchable-snapshots/qa/gcs/src/test/java/org/elasticsearch/xpack/searchablesnapshots/GCSSearchableSnapshotsIT.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@
1515
public class GCSSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
1616

1717
@Override
18-
protected String repositoryType() {
18+
protected String writeRepositoryType() {
1919
return "gcs";
2020
}
2121

2222
@Override
23-
protected Settings repositorySettings() {
23+
protected Settings writeRepositorySettings() {
2424
final String bucket = System.getProperty("test.gcs.bucket");
2525
assertThat(bucket, not(blankOrNullString()));
2626

‎x-pack/plugin/searchable-snapshots/qa/hdfs/src/test/java/org/elasticsearch/xpack/searchablesnapshots/hdfs/HdfsSearchableSnapshotsIT.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@
1515

1616
public class HdfsSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
1717
@Override
18-
protected String repositoryType() {
18+
protected String writeRepositoryType() {
1919
return "hdfs";
2020
}
2121

2222
@Override
23-
protected Settings repositorySettings() {
23+
protected Settings writeRepositorySettings() {
2424
final String uri = System.getProperty("test.hdfs.uri");
2525
assertThat(uri, not(blankOrNullString()));
2626

‎x-pack/plugin/searchable-snapshots/qa/minio/src/test/java/org/elasticsearch/xpack/searchablesnapshots/minio/MinioSearchableSnapshotsIT.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@
1515
public class MinioSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
1616

1717
@Override
18-
protected String repositoryType() {
18+
protected String writeRepositoryType() {
1919
return "s3";
2020
}
2121

2222
@Override
23-
protected Settings repositorySettings() {
23+
protected Settings writeRepositorySettings() {
2424
final String bucket = System.getProperty("test.minio.bucket");
2525
assertThat(bucket, not(blankOrNullString()));
2626

‎x-pack/plugin/searchable-snapshots/qa/rest/src/test/java/org/elasticsearch/xpack/searchablesnapshots/rest/FsSearchableSnapshotsIT.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,12 @@
1313
public class FsSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase {
1414

1515
@Override
16-
protected String repositoryType() {
16+
protected String writeRepositoryType() {
1717
return FsRepository.TYPE;
1818
}
1919

2020
@Override
21-
protected Settings repositorySettings() {
21+
protected Settings writeRepositorySettings() {
2222
return Settings.builder().put("location", System.getProperty("tests.path.repo")).build();
2323
}
2424
}

‎x-pack/plugin/searchable-snapshots/qa/rest/src/test/resources/rest-api-spec/test/url_repo.yml

Lines changed: 0 additions & 21 deletions
This file was deleted.

0 commit comments

Comments
 (0)