Skip to content

Commit 719e75b

Browse files
Tim-BrooksTim-Brooks
authored
Add repository-url module and move URLRepository (#22752)
This is related to #22116. URLRepository requires SocketPermission connect. This commit introduces a new module called "repository-url" where URLRepository will reside. With the new module, permissions can be removed from core.
1 parent e9a68b3 commit 719e75b

File tree

21 files changed

+525
-132
lines changed

21 files changed

+525
-132
lines changed

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -276,8 +276,7 @@ class ClusterFormationTasks {
276276
'path.repo' : "${node.sharedDir}/repo",
277277
'path.shared_data' : "${node.sharedDir}/",
278278
// Define a node attribute so we can test that it exists
279-
'node.attr.testattr' : 'test',
280-
'repositories.url.allowed_urls': 'http://snapshot.test*'
279+
'node.attr.testattr' : 'test'
281280
]
282281
// we set min master nodes to the total number of nodes in the cluster and
283282
// basically skip initial state recovery to allow the cluster to form using a realistic master election

‎buildSrc/src/main/resources/checkstyle_suppressions.xml

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,6 @@
256256
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Base64.java" checks="LineLength" />
257257
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]Numbers.java" checks="LineLength" />
258258
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]fs[/\\]FsBlobStore.java" checks="LineLength" />
259-
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]blobstore[/\\]url[/\\]URLBlobStore.java" checks="LineLength" />
260259
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]BytesArray.java" checks="LineLength" />
261260
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]bytes[/\\]PagedBytesReference.java" checks="LineLength" />
262261
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]common[/\\]cache[/\\]Cache.java" checks="LineLength" />
@@ -437,7 +436,6 @@
437436
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]blobstore[/\\]ChecksumBlobStoreFormat.java" checks="LineLength" />
438437
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]fs[/\\]FsRepository.java" checks="LineLength" />
439438
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLIndexShardRepository.java" checks="LineLength" />
440-
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]repositories[/\\]uri[/\\]URLRepository.java" checks="LineLength" />
441439
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]RestController.java" checks="LineLength" />
442440
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestCountAction.java" checks="LineLength" />
443441
<suppress files="core[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]rest[/\\]action[/\\]cat[/\\]RestIndicesAction.java" checks="LineLength" />

‎core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,6 @@
8181
import org.elasticsearch.node.Node;
8282
import org.elasticsearch.plugins.PluginsService;
8383
import org.elasticsearch.repositories.fs.FsRepository;
84-
import org.elasticsearch.repositories.uri.URLRepository;
8584
import org.elasticsearch.rest.BaseRestHandler;
8685
import org.elasticsearch.script.ScriptService;
8786
import org.elasticsearch.search.SearchModule;
@@ -348,9 +347,6 @@ public void apply(Settings value, Settings current, Settings previous) {
348347
Node.NODE_INGEST_SETTING,
349348
Node.NODE_ATTRIBUTES,
350349
Node.NODE_LOCAL_STORAGE_SETTING,
351-
URLRepository.ALLOWED_URLS_SETTING,
352-
URLRepository.REPOSITORIES_URL_SETTING,
353-
URLRepository.SUPPORTED_PROTOCOLS_SETTING,
354350
TransportMasterNodeReadAction.FORCE_LOCAL_SETTING,
355351
AutoCreateIndex.AUTO_CREATE_INDEX_SETTING,
356352
BaseRestHandler.MULTI_ALLOW_EXPLICIT_INDEX,

‎core/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,24 +19,22 @@
1919

2020
package org.elasticsearch.repositories;
2121

22-
import java.util.Collections;
23-
import java.util.HashMap;
24-
import java.util.List;
25-
import java.util.Map;
26-
2722
import org.elasticsearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus;
2823
import org.elasticsearch.common.inject.AbstractModule;
29-
import org.elasticsearch.common.inject.binder.LinkedBindingBuilder;
3024
import org.elasticsearch.common.inject.multibindings.MapBinder;
3125
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
3226
import org.elasticsearch.env.Environment;
3327
import org.elasticsearch.plugins.RepositoryPlugin;
3428
import org.elasticsearch.repositories.fs.FsRepository;
35-
import org.elasticsearch.repositories.uri.URLRepository;
3629
import org.elasticsearch.snapshots.RestoreService;
3730
import org.elasticsearch.snapshots.SnapshotShardsService;
3831
import org.elasticsearch.snapshots.SnapshotsService;
3932

33+
import java.util.Collections;
34+
import java.util.HashMap;
35+
import java.util.List;
36+
import java.util.Map;
37+
4038
/**
4139
* Sets up classes for Snapshot/Restore.
4240
*/
@@ -47,7 +45,6 @@ public class RepositoriesModule extends AbstractModule {
4745
public RepositoriesModule(Environment env, List<RepositoryPlugin> repoPlugins, NamedXContentRegistry namedXContentRegistry) {
4846
Map<String, Repository.Factory> factories = new HashMap<>();
4947
factories.put(FsRepository.TYPE, (metadata) -> new FsRepository(metadata, env, namedXContentRegistry));
50-
factories.put(URLRepository.TYPE, (metadata) -> new URLRepository(metadata, env, namedXContentRegistry));
5148

5249
for (RepositoryPlugin repoPlugin : repoPlugins) {
5350
Map<String, Repository.Factory> newRepoTypes = repoPlugin.getRepositories(env, namedXContentRegistry);

‎core/src/test/java/org/elasticsearch/bwcompat/RestoreBackwardsCompatIT.java

Lines changed: 21 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -26,21 +26,22 @@
2626
import org.elasticsearch.cluster.metadata.IndexMetaData;
2727
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
2828
import org.elasticsearch.cluster.routing.allocation.decider.FilterAllocationDecider;
29+
import org.elasticsearch.common.io.FileTestUtils;
2930
import org.elasticsearch.common.settings.Settings;
3031
import org.elasticsearch.env.Environment;
31-
import org.elasticsearch.repositories.uri.URLRepository;
32+
import org.elasticsearch.repositories.fs.FsRepository;
3233
import org.elasticsearch.rest.RestStatus;
3334
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
3435
import org.elasticsearch.snapshots.RestoreInfo;
3536
import org.elasticsearch.snapshots.SnapshotInfo;
3637
import org.elasticsearch.snapshots.SnapshotRestoreException;
38+
import org.elasticsearch.snapshots.mockstore.MockRepository;
3739
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
3840
import org.elasticsearch.test.ESIntegTestCase.Scope;
3941
import org.elasticsearch.test.VersionUtils;
42+
import org.junit.BeforeClass;
4043

4144
import java.io.IOException;
42-
import java.net.URI;
43-
import java.net.URISyntaxException;
4445
import java.nio.file.DirectoryStream;
4546
import java.nio.file.Files;
4647
import java.nio.file.Path;
@@ -61,27 +62,19 @@
6162
@ClusterScope(scope = Scope.TEST)
6263
public class RestoreBackwardsCompatIT extends AbstractSnapshotIntegTestCase {
6364

65+
private static Path repoPath;
66+
6467
@Override
6568
protected Settings nodeSettings(int nodeOrdinal) {
66-
if (randomBoolean()) {
67-
// Configure using path.repo
68-
return Settings.builder()
69-
.put(super.nodeSettings(nodeOrdinal))
70-
.put(Environment.PATH_REPO_SETTING.getKey(), getBwcIndicesPath())
71-
.build();
72-
} else {
73-
// Configure using url white list
74-
try {
75-
URI repoJarPatternUri = new URI("jar:" + getBwcIndicesPath().toUri().toString() + "*.zip!/repo/");
76-
return Settings.builder()
77-
.put(super.nodeSettings(nodeOrdinal))
78-
.putArray(URLRepository.ALLOWED_URLS_SETTING.getKey(), repoJarPatternUri.toString())
79-
.build();
80-
} catch (URISyntaxException ex) {
81-
throw new IllegalArgumentException(ex);
82-
}
69+
return Settings.builder()
70+
.put(super.nodeSettings(nodeOrdinal))
71+
.put(Environment.PATH_REPO_SETTING.getKey(), repoPath)
72+
.build();
73+
}
8374

84-
}
75+
@BeforeClass
76+
public static void repoSetup() throws IOException {
77+
repoPath = createTempDir("repositories");
8578
}
8679

8780
public void testRestoreOldSnapshots() throws Exception {
@@ -151,13 +144,14 @@ private List<String> listRepoVersions(String prefix) throws Exception {
151144
}
152145

153146
private void createRepo(String prefix, String version, String repo) throws Exception {
154-
Path repoFile = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip");
155-
URI repoFileUri = repoFile.toUri();
156-
URI repoJarUri = new URI("jar:" + repoFileUri.toString() + "!/repo/");
147+
Path repoFileFromBuild = getBwcIndicesPath().resolve(prefix + "-" + version + ".zip");
148+
String repoFileName = repoFileFromBuild.getFileName().toString().split(".zip")[0];
149+
Path fsRepoPath = repoPath.resolve(repoFileName);
150+
FileTestUtils.unzip(repoFileFromBuild, fsRepoPath, null);
157151
logger.info("--> creating repository [{}] for version [{}]", repo, version);
158152
assertAcked(client().admin().cluster().preparePutRepository(repo)
159-
.setType("url").setSettings(Settings.builder()
160-
.put("url", repoJarUri.toString())));
153+
.setType(MockRepository.TYPE).setSettings(Settings.builder()
154+
.put(FsRepository.REPOSITORIES_LOCATION_SETTING.getKey(), fsRepoPath.getParent().relativize(fsRepoPath).resolve("repo").toString())));
161155
}
162156

163157
private void testOldSnapshot(String version, String repo, String snapshot) throws IOException {
@@ -198,7 +192,7 @@ private void testOldSnapshot(String version, String repo, String snapshot) throw
198192
equalTo("{\"type1\":{\"_source\":{\"enabled\":0}}}"),
199193
equalTo("{\"type1\":{\"_source\":{\"enabled\":\"off\"}}}"),
200194
equalTo("{\"type1\":{\"_source\":{\"enabled\":\"no\"}}}")
201-
));
195+
));
202196
assertThat(template.aliases().size(), equalTo(3));
203197
assertThat(template.aliases().get("alias1"), notNullValue());
204198
assertThat(template.aliases().get("alias2").filter().string(), containsString(version));

‎core/src/test/java/org/elasticsearch/snapshots/RepositoriesIT.java

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -142,30 +142,6 @@ public void testMisconfiguredRepository() throws Exception {
142142
} catch (RepositoryException ex) {
143143
assertThat(ex.toString(), containsString("location [" + location + "] doesn't match any of the locations specified by path.repo"));
144144
}
145-
146-
String repoUrl = invalidRepoPath.toAbsolutePath().toUri().toURL().toString();
147-
String unsupportedUrl = repoUrl.replace("file:/", "netdoc:/");
148-
logger.info("--> trying creating url repository with unsupported url protocol");
149-
try {
150-
client().admin().cluster().preparePutRepository("test-repo")
151-
.setType("url").setSettings(Settings.builder().put("url", unsupportedUrl))
152-
.get();
153-
fail("Shouldn't be here");
154-
} catch (RepositoryException ex) {
155-
assertThat(ex.toString(),
156-
either(containsString("unsupported url protocol [netdoc]"))
157-
.or(containsString("unknown protocol: netdoc"))); // newer versions of JDK 9
158-
}
159-
160-
logger.info("--> trying creating url repository with location that is not registered in path.repo setting");
161-
try {
162-
client().admin().cluster().preparePutRepository("test-repo")
163-
.setType("url").setSettings(Settings.builder().put("url", invalidRepoPath.toUri().toURL()))
164-
.get();
165-
fail("Shouldn't be here");
166-
} catch (RepositoryException ex) {
167-
assertThat(ex.toString(), containsString("doesn't match any of the locations specified by path.repo"));
168-
}
169145
}
170146

171147
public void testRepositoryAckTimeout() throws Exception {

‎core/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java

Lines changed: 0 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1442,63 +1442,6 @@ public void testDeleteRepositoryWhileSnapshotting() throws Exception {
14421442
assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
14431443
}
14441444

1445-
public void testUrlRepository() throws Exception {
1446-
Client client = client();
1447-
1448-
logger.info("--> creating repository");
1449-
Path repositoryLocation = randomRepoPath();
1450-
assertAcked(client.admin().cluster().preparePutRepository("test-repo")
1451-
.setType("fs").setSettings(Settings.builder()
1452-
.put("location", repositoryLocation)
1453-
.put("compress", randomBoolean())
1454-
.put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
1455-
1456-
createIndex("test-idx");
1457-
ensureGreen();
1458-
1459-
logger.info("--> indexing some data");
1460-
for (int i = 0; i < 100; i++) {
1461-
index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
1462-
}
1463-
refresh();
1464-
assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
1465-
1466-
logger.info("--> snapshot");
1467-
CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
1468-
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
1469-
assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
1470-
1471-
assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
1472-
1473-
logger.info("--> delete index");
1474-
cluster().wipeIndices("test-idx");
1475-
1476-
logger.info("--> create read-only URL repository");
1477-
assertAcked(client.admin().cluster().preparePutRepository("url-repo")
1478-
.setType("url").setSettings(Settings.builder()
1479-
.put("url", repositoryLocation.toUri().toURL())
1480-
.put("list_directories", randomBoolean())));
1481-
logger.info("--> restore index after deletion");
1482-
RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("url-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute().actionGet();
1483-
assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
1484-
1485-
assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().totalHits(), equalTo(100L));
1486-
1487-
logger.info("--> list available shapshots");
1488-
GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
1489-
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
1490-
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
1491-
1492-
logger.info("--> delete snapshot");
1493-
DeleteSnapshotResponse deleteSnapshotResponse = client.admin().cluster().prepareDeleteSnapshot("test-repo", "test-snap").get();
1494-
assertAcked(deleteSnapshotResponse);
1495-
1496-
logger.info("--> list available shapshot again, no snapshots should be returned");
1497-
getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("url-repo").get();
1498-
assertThat(getSnapshotsResponse.getSnapshots(), notNullValue());
1499-
assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(0));
1500-
}
1501-
15021445
@TestLogging("_root:DEBUG") // this fails every now and then: https://github.com/elastic/elasticsearch/issues/18121 but without
15031446
// more logs we cannot find out why
15041447
public void testReadonlyRepository() throws Exception {

‎modules/repository-url/build.gradle

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
esplugin {
21+
description 'Module for URL repository'
22+
classname 'org.elasticsearch.plugin.repository.url.URLRepositoryPlugin'
23+
}
24+
25+
integTest {
26+
cluster {
27+
setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
28+
}
29+
}

‎core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java renamed to ‎modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobContainer.java

File renamed without changes.

‎core/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java renamed to ‎modules/repository-url/src/main/java/org/elasticsearch/common/blobstore/url/URLBlobStore.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ public class URLBlobStore extends AbstractComponent implements BlobStore {
5555
public URLBlobStore(Settings settings, URL path) {
5656
super(settings);
5757
this.path = path;
58-
this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
58+
this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size",
59+
new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes();
5960
}
6061

6162
/**

‎modules/repository-url/src/main/java/org/elasticsearch/plugin/repository/url/URLRepositoryPlugin.java

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.plugin.repository.url;
21+
22+
import org.elasticsearch.common.settings.Setting;
23+
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
24+
import org.elasticsearch.env.Environment;
25+
import org.elasticsearch.plugins.Plugin;
26+
import org.elasticsearch.plugins.RepositoryPlugin;
27+
import org.elasticsearch.repositories.Repository;
28+
import org.elasticsearch.repositories.url.URLRepository;
29+
30+
import java.util.Arrays;
31+
import java.util.Collections;
32+
import java.util.List;
33+
import java.util.Map;
34+
35+
public class URLRepositoryPlugin extends Plugin implements RepositoryPlugin {
36+
37+
@Override
38+
public List<Setting<?>> getSettings() {
39+
return Arrays.asList(
40+
URLRepository.ALLOWED_URLS_SETTING,
41+
URLRepository.REPOSITORIES_URL_SETTING,
42+
URLRepository.SUPPORTED_PROTOCOLS_SETTING
43+
);
44+
}
45+
46+
@Override
47+
public Map<String, Repository.Factory> getRepositories(Environment env, NamedXContentRegistry namedXContentRegistry) {
48+
return Collections.singletonMap(URLRepository.TYPE, metadata -> new URLRepository(metadata, env, namedXContentRegistry));
49+
}
50+
}

0 commit comments

Comments
 (0)