Skip to content

Commit 46e0576

Browse files
committed
Use fixture to test repository-url module
Pursuing the effort to test repository implementations using fixtures, this commit adds a YAML integration test for the repository-url module that uses a fixture to test URL based repositories on both http:// and file:// prefixes.
1 parent 4b1ed20 commit 46e0576

File tree

4 files changed

+479
-19
lines changed

4 files changed

+479
-19
lines changed

modules/repository-url/build.gradle

+26-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import org.elasticsearch.gradle.test.AntFixture
2+
13
/*
24
* Licensed to Elasticsearch under one or more contributor
35
* license agreements. See the NOTICE file distributed with
@@ -22,6 +24,29 @@ esplugin {
2224
classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
2325
}
2426

27+
forbiddenApisTest {
28+
// we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
29+
bundledSignatures -= 'jdk-non-portable'
30+
bundledSignatures += 'jdk-internal'
31+
}
32+
33+
// This directory is shared between two URL repositories and one FS repository in YAML integration tests
34+
File repositoryDir = new File(project.buildDir, "shared-repository")
35+
36+
/** A task to start the URLFixture that exposes the repositoryDir over HTTP **/
37+
task urlFixture(type: AntFixture) {
38+
doFirst {
39+
repositoryDir.mkdirs()
40+
}
41+
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
42+
executable = new File(project.runtimeJavaHome, 'bin/java')
43+
args 'org.elasticsearch.repositories.url.URLFixture', baseDir, "${repositoryDir.absolutePath}"
44+
}
45+
2546
integTestCluster {
26-
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
47+
dependsOn urlFixture
48+
// repositoryDir is used by a FS repository to create snapshots
49+
setting 'path.repo', "${repositoryDir.absolutePath}"
50+
// repositoryDir is used by a URL repository to restore snapshots
51+
setting 'repositories.url.allowed_urls', "http://snapshot.test*,http://${ -> urlFixture.addressAndPort }"
2752
}

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

+77-1
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,31 @@
2121

2222
import com.carrotsearch.randomizedtesting.annotations.Name;
2323
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
24-
24+
import org.apache.http.HttpEntity;
25+
import org.apache.http.entity.ContentType;
26+
import org.apache.http.nio.entity.NStringEntity;
27+
import org.elasticsearch.client.Response;
28+
import org.elasticsearch.common.Strings;
29+
import org.elasticsearch.common.settings.Settings;
30+
import org.elasticsearch.common.xcontent.ToXContent;
31+
import org.elasticsearch.common.xcontent.XContentBuilder;
32+
import org.elasticsearch.common.xcontent.support.XContentMapValues;
33+
import org.elasticsearch.repositories.fs.FsRepository;
34+
import org.elasticsearch.rest.RestStatus;
2535
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
2636
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
37+
import org.junit.Before;
38+
39+
import java.io.IOException;
40+
import java.net.InetAddress;
41+
import java.net.URL;
42+
import java.util.List;
43+
import java.util.Map;
44+
45+
import static java.util.Collections.emptyMap;
46+
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
47+
import static org.hamcrest.Matchers.equalTo;
48+
import static org.hamcrest.Matchers.hasSize;
2749

2850
public class RepositoryURLClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
2951

@@ -35,5 +57,59 @@ public RepositoryURLClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate
3557
public static Iterable<Object[]> parameters() throws Exception {
3658
return ESClientYamlSuiteTestCase.createParameters();
3759
}
60+
61+
@Before
62+
public void registerRepositories() throws IOException {
63+
Response clusterSettingsResponse = client().performRequest("GET", "/_cluster/settings?include_defaults=true" +
64+
"&filter_path=defaults.path.repo,defaults.repositories.url.allowed_urls");
65+
Map<String, Object> clusterSettings = entityAsMap(clusterSettingsResponse);
66+
67+
@SuppressWarnings("unchecked")
68+
List<String> pathRepo = (List<String>) XContentMapValues.extractValue("defaults.path.repo", clusterSettings);
69+
assertThat(pathRepo, hasSize(1));
70+
71+
// Create a FS repository using the path.repo location
72+
Response createFsRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-fs", emptyMap(),
73+
buildRepositorySettings(FsRepository.TYPE, Settings.builder().put("location", pathRepo.get(0)).build()));
74+
assertThat(createFsRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
75+
76+
// Create a URL repository using the file://{path.repo} URL
77+
Response createFileRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-file", emptyMap(),
78+
buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", "file://" + pathRepo.get(0)).build()));
79+
assertThat(createFileRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
80+
81+
// Create a URL repository using the http://{fixture} URL
82+
@SuppressWarnings("unchecked")
83+
List<String> allowedUrls = (List<String>) XContentMapValues.extractValue("defaults.repositories.url.allowed_urls", clusterSettings);
84+
for (String allowedUrl : allowedUrls) {
85+
try {
86+
InetAddress inetAddress = InetAddress.getByName(new URL(allowedUrl).getHost());
87+
if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress()) {
88+
Response createUrlRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-url", emptyMap(),
89+
buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", allowedUrl).build()));
90+
assertThat(createUrlRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus()));
91+
break;
92+
}
93+
} catch (Exception e) {
94+
logger.debug("Failed to resolve inet address for allowed URL [{}], skipping", allowedUrl);
95+
}
96+
}
97+
}
98+
99+
private static HttpEntity buildRepositorySettings(final String type, final Settings settings) throws IOException {
100+
try (XContentBuilder builder = jsonBuilder()) {
101+
builder.startObject();
102+
{
103+
builder.field("type", type);
104+
builder.startObject("settings");
105+
{
106+
settings.toXContent(builder, ToXContent.EMPTY_PARAMS);
107+
}
108+
builder.endObject();
109+
}
110+
builder.endObject();
111+
return new NStringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON);
112+
}
113+
}
38114
}
39115

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.elasticsearch.repositories.url;
20+
21+
import com.sun.net.httpserver.HttpExchange;
22+
import com.sun.net.httpserver.HttpHandler;
23+
import com.sun.net.httpserver.HttpServer;
24+
import org.elasticsearch.common.SuppressForbidden;
25+
import org.elasticsearch.mocksocket.MockHttpServer;
26+
import org.elasticsearch.rest.RestStatus;
27+
28+
import java.io.IOException;
29+
import java.lang.management.ManagementFactory;
30+
import java.net.Inet6Address;
31+
import java.net.InetAddress;
32+
import java.net.InetSocketAddress;
33+
import java.net.SocketAddress;
34+
import java.nio.file.Files;
35+
import java.nio.file.Path;
36+
import java.nio.file.Paths;
37+
import java.nio.file.StandardCopyOption;
38+
import java.util.Map;
39+
import java.util.Objects;
40+
41+
import static java.util.Collections.emptyMap;
42+
import static java.util.Collections.singleton;
43+
import static java.util.Collections.singletonMap;
44+
45+
/**
46+
* This {@link URLFixture} exposes a filesystem directory over HTTP. It is used in repository-url
47+
* integration tests to expose a directory populated using a regular FS repository.
48+
*/
49+
public class URLFixture {
50+
51+
public static void main(String[] args) throws Exception {
52+
if (args == null || args.length != 2) {
53+
throw new IllegalArgumentException("URLFixture <working directory> <repository directory>");
54+
}
55+
56+
final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0);
57+
final HttpServer httpServer = MockHttpServer.createHttp(socketAddress, 0);
58+
59+
try {
60+
final Path workingDirectory = dir(args[0]);
61+
/// Writes the PID of the current Java process in a `pid` file located in the working directory
62+
writeFile(workingDirectory, "pid", ManagementFactory.getRuntimeMXBean().getName().split("@")[0]);
63+
64+
final String addressAndPort = addressToString(httpServer.getAddress());
65+
// Writes the address and port of the http server in a `ports` file located in the working directory
66+
writeFile(workingDirectory, "ports", addressAndPort);
67+
68+
// Exposes the repository over HTTP
69+
final String url = "http://" + addressAndPort;
70+
httpServer.createContext("/", new ResponseHandler(dir(args[1])));
71+
httpServer.start();
72+
73+
// Wait to be killed
74+
Thread.sleep(Long.MAX_VALUE);
75+
76+
} finally {
77+
httpServer.stop(0);
78+
}
79+
}
80+
81+
@SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
82+
private static Path dir(final String dir) {
83+
return Paths.get(dir);
84+
}
85+
86+
private static void writeFile(final Path dir, final String fileName, final String content) throws IOException {
87+
final Path tempPidFile = Files.createTempFile(dir, null, null);
88+
Files.write(tempPidFile, singleton(content));
89+
Files.move(tempPidFile, dir.resolve(fileName), StandardCopyOption.ATOMIC_MOVE);
90+
}
91+
92+
private static String addressToString(final SocketAddress address) {
93+
final InetSocketAddress inetSocketAddress = (InetSocketAddress) address;
94+
if (inetSocketAddress.getAddress() instanceof Inet6Address) {
95+
return "[" + inetSocketAddress.getHostString() + "]:" + inetSocketAddress.getPort();
96+
} else {
97+
return inetSocketAddress.getHostString() + ":" + inetSocketAddress.getPort();
98+
}
99+
}
100+
101+
static class ResponseHandler implements HttpHandler {
102+
103+
private final Path repositoryDir;
104+
105+
ResponseHandler(final Path repositoryDir) {
106+
this.repositoryDir = repositoryDir;
107+
}
108+
109+
@Override
110+
public void handle(HttpExchange exchange) throws IOException {
111+
Response response;
112+
if ("GET".equalsIgnoreCase(exchange.getRequestMethod())) {
113+
String path = exchange.getRequestURI().toString();
114+
if (path.length() > 0 && path.charAt(0) == '/') {
115+
path = path.substring(1);
116+
}
117+
118+
Path normalizedRepositoryDir = repositoryDir.normalize();
119+
Path normalizedPath = normalizedRepositoryDir.resolve(path).normalize();
120+
121+
if (normalizedPath.startsWith(normalizedRepositoryDir)) {
122+
if (Files.exists(normalizedPath) && Files.isReadable(normalizedPath) && Files.isRegularFile(normalizedPath)) {
123+
byte[] content = Files.readAllBytes(normalizedPath);
124+
Map<String, String> headers = singletonMap("Content-Length", String.valueOf(content.length));
125+
response = new Response(RestStatus.OK, headers, "application/octet-stream", content);
126+
} else {
127+
response = new Response(RestStatus.NOT_FOUND, emptyMap(), "text/plain", new byte[0]);
128+
}
129+
} else {
130+
response = new Response(RestStatus.FORBIDDEN, emptyMap(), "text/plain", new byte[0]);
131+
}
132+
} else {
133+
response = new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), "text/plain", new byte[0]);
134+
}
135+
exchange.sendResponseHeaders(response.status.getStatus(), response.body.length);
136+
if (response.body.length > 0) {
137+
exchange.getResponseBody().write(response.body);
138+
}
139+
exchange.close();
140+
}
141+
}
142+
143+
/**
144+
* Represents a HTTP Response.
145+
*/
146+
static class Response {
147+
148+
final RestStatus status;
149+
final Map<String, String> headers;
150+
final String contentType;
151+
final byte[] body;
152+
153+
Response(final RestStatus status, final Map<String, String> headers, final String contentType, final byte[] body) {
154+
this.status = Objects.requireNonNull(status);
155+
this.headers = Objects.requireNonNull(headers);
156+
this.contentType = Objects.requireNonNull(contentType);
157+
this.body = Objects.requireNonNull(body);
158+
}
159+
}
160+
}

0 commit comments

Comments
 (0)