Skip to content

Commit 6509c59

Browse files
author
Vladimir Dolzhenko
committed
lazy snapshot repository initialization (#31606)
(cherry picked from commit b1bf643)
1 parent 3c88e66 commit 6509c59

File tree

23 files changed

+540
-216
lines changed

23 files changed

+540
-216
lines changed

‎modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java

+21-10
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.repositories.url;
2121

2222
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
23+
import org.elasticsearch.common.blobstore.BlobContainer;
2324
import org.elasticsearch.common.blobstore.BlobPath;
2425
import org.elasticsearch.common.blobstore.BlobStore;
2526
import org.elasticsearch.common.blobstore.url.URLBlobStore;
@@ -31,7 +32,6 @@
3132
import org.elasticsearch.repositories.RepositoryException;
3233
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
3334

34-
import java.io.IOException;
3535
import java.net.MalformedURLException;
3636
import java.net.URISyntaxException;
3737
import java.net.URL;
@@ -71,33 +71,44 @@ public class URLRepository extends BlobStoreRepository {
7171

7272
private final Environment environment;
7373

74-
private final URLBlobStore blobStore;
75-
7674
private final BlobPath basePath;
7775

76+
private final URL url;
77+
7878
/**
7979
* Constructs a read-only URL-based repository
8080
*/
8181
public URLRepository(RepositoryMetaData metadata, Environment environment,
82-
NamedXContentRegistry namedXContentRegistry) throws IOException {
82+
NamedXContentRegistry namedXContentRegistry) {
8383
super(metadata, environment.settings(), namedXContentRegistry);
8484

8585
if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) == false) {
8686
throw new RepositoryException(metadata.name(), "missing url");
8787
}
88+
this.environment = environment;
8889
supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(settings);
8990
urlWhiteList = ALLOWED_URLS_SETTING.get(settings).toArray(new URIPattern[]{});
90-
this.environment = environment;
91+
basePath = BlobPath.cleanPath();
92+
url = URL_SETTING.exists(metadata.settings())
93+
? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
94+
}
9195

92-
URL url = URL_SETTING.exists(metadata.settings()) ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
96+
@Override
97+
protected BlobStore createBlobStore() {
9398
URL normalizedURL = checkURL(url);
94-
blobStore = new URLBlobStore(settings, normalizedURL);
95-
basePath = BlobPath.cleanPath();
99+
return new URLBlobStore(settings, normalizedURL);
100+
}
101+
102+
// only use for testing
103+
@Override
104+
protected BlobContainer blobContainer() {
105+
return super.blobContainer();
96106
}
97107

108+
// only use for testing
98109
@Override
99-
protected BlobStore blobStore() {
100-
return blobStore;
110+
protected BlobStore getBlobStore() {
111+
return super.getBlobStore();
101112
}
102113

103114
@Override

‎modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java

+45-6
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,22 @@
3131
import java.nio.file.Path;
3232
import java.util.Collections;
3333

34+
import static org.hamcrest.CoreMatchers.is;
35+
import static org.hamcrest.CoreMatchers.not;
36+
import static org.hamcrest.CoreMatchers.nullValue;
37+
3438
public class URLRepositoryTests extends ESTestCase {
3539

40+
private URLRepository createRepository(Settings baseSettings, RepositoryMetaData repositoryMetaData) {
41+
return new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
42+
new NamedXContentRegistry(Collections.emptyList())) {
43+
@Override
44+
protected void assertSnapshotOrGenericThread() {
45+
// eliminate thread name check as we create repo manually on test/main threads
46+
}
47+
};
48+
}
49+
3650
public void testWhiteListingRepoURL() throws IOException {
3751
String repoPath = createTempDir().resolve("repository").toUri().toURL().toString();
3852
Settings baseSettings = Settings.builder()
@@ -41,8 +55,12 @@ public void testWhiteListingRepoURL() throws IOException {
4155
.put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
4256
.build();
4357
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
44-
new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
45-
new NamedXContentRegistry(Collections.emptyList()));
58+
final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
59+
repository.start();
60+
61+
assertThat("blob store has to be lazy initialized", repository.getBlobStore(), is(nullValue()));
62+
repository.blobContainer();
63+
assertThat("blobContainer has to initialize blob store", repository.getBlobStore(), not(nullValue()));
4664
}
4765

4866
public void testIfNotWhiteListedMustSetRepoURL() throws IOException {
@@ -52,9 +70,10 @@ public void testIfNotWhiteListedMustSetRepoURL() throws IOException {
5270
.put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
5371
.build();
5472
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
73+
final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
74+
repository.start();
5575
try {
56-
new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
57-
new NamedXContentRegistry(Collections.emptyList()));
76+
repository.blobContainer();
5877
fail("RepositoryException should have been thrown.");
5978
} catch (RepositoryException e) {
6079
String msg = "[url] file url [" + repoPath
@@ -73,13 +92,33 @@ public void testMustBeSupportedProtocol() throws IOException {
7392
.put(URLRepository.SUPPORTED_PROTOCOLS_SETTING.getKey(), "http,https")
7493
.build();
7594
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
95+
final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
96+
repository.start();
7697
try {
77-
new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
78-
new NamedXContentRegistry(Collections.emptyList()));
98+
repository.blobContainer();
7999
fail("RepositoryException should have been thrown.");
80100
} catch (RepositoryException e) {
81101
assertEquals("[url] unsupported url protocol [file] from URL [" + repoPath +"]", e.getMessage());
82102
}
83103
}
84104

105+
public void testNonNormalizedUrl() throws IOException {
106+
Settings baseSettings = Settings.builder()
107+
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
108+
.put(URLRepository.ALLOWED_URLS_SETTING.getKey(), "file:/tmp/")
109+
.put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), "file:/var/" )
110+
.build();
111+
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
112+
final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
113+
repository.start();
114+
try {
115+
repository.blobContainer();
116+
fail("RepositoryException should have been thrown.");
117+
} catch (RepositoryException e) {
118+
assertEquals("[url] file url [file:/var/] doesn't match any of the locations "
119+
+ "specified by path.repo or repositories.url.allowed_urls",
120+
e.getMessage());
121+
}
122+
}
123+
85124
}

‎plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

+28-14
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,6 @@
4040
import org.elasticsearch.snapshots.SnapshotCreationException;
4141
import org.elasticsearch.snapshots.SnapshotId;
4242

43-
import java.io.IOException;
4443
import java.net.URISyntaxException;
4544
import java.util.List;
4645
import java.util.Locale;
@@ -80,25 +79,21 @@ public static final class Repository {
8079
public static final Setting<Boolean> READONLY_SETTING = Setting.boolSetting("readonly", false, Property.NodeScope);
8180
}
8281

83-
private final AzureBlobStore blobStore;
8482
private final BlobPath basePath;
8583
private final ByteSizeValue chunkSize;
8684
private final boolean compress;
85+
private final Environment environment;
86+
private final AzureStorageService storageService;
8787
private final boolean readonly;
8888

8989
public AzureRepository(RepositoryMetaData metadata, Environment environment, NamedXContentRegistry namedXContentRegistry,
90-
AzureStorageService storageService) throws IOException, URISyntaxException, StorageException {
90+
AzureStorageService storageService) {
9191
super(metadata, environment.settings(), namedXContentRegistry);
92-
this.blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
9392
this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings());
9493
this.compress = Repository.COMPRESS_SETTING.get(metadata.settings());
95-
// If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
96-
// For secondary_only setting, the repository should be read only
97-
if (Repository.READONLY_SETTING.exists(metadata.settings())) {
98-
this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
99-
} else {
100-
this.readonly = this.blobStore.getLocationMode() == LocationMode.SECONDARY_ONLY;
101-
}
94+
this.environment = environment;
95+
this.storageService = storageService;
96+
10297
final String basePath = Strings.trimLeadingCharacter(Repository.BASE_PATH_SETTING.get(metadata.settings()), '/');
10398
if (Strings.hasLength(basePath)) {
10499
// Remove starting / if any
@@ -110,15 +105,33 @@ public AzureRepository(RepositoryMetaData metadata, Environment environment, Nam
110105
} else {
111106
this.basePath = BlobPath.cleanPath();
112107
}
113-
logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
114-
"using container [{}], chunk_size [{}], compress [{}], base_path [{}]", blobStore, chunkSize, compress, basePath));
108+
109+
// If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
110+
// For secondary_only setting, the repository should be read only
111+
final LocationMode locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
112+
if (Repository.READONLY_SETTING.exists(metadata.settings())) {
113+
this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
114+
} else {
115+
this.readonly = locationMode == LocationMode.SECONDARY_ONLY;
116+
}
117+
}
118+
119+
// only use for testing
120+
@Override
121+
protected BlobStore getBlobStore() {
122+
return super.getBlobStore();
115123
}
116124

117125
/**
118126
* {@inheritDoc}
119127
*/
120128
@Override
121-
protected BlobStore blobStore() {
129+
protected AzureBlobStore createBlobStore() throws URISyntaxException, StorageException {
130+
final AzureBlobStore blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
131+
132+
logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
133+
"using container [{}], chunk_size [{}], compress [{}], base_path [{}]",
134+
blobStore, chunkSize, compress, basePath));
122135
return blobStore;
123136
}
124137

@@ -146,6 +159,7 @@ protected ByteSizeValue chunkSize() {
146159
@Override
147160
public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData clusterMetadata) {
148161
try {
162+
final AzureBlobStore blobStore = (AzureBlobStore) blobStore();
149163
if (blobStore.containerExist() == false) {
150164
throw new IllegalArgumentException("The bucket [" + blobStore + "] does not exist. Please create it before "
151165
+ " creating an azure snapshot repository backed by it.");

‎plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java

+14-16
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
package org.elasticsearch.repositories.azure;
2121

2222
import com.microsoft.azure.storage.LocationMode;
23-
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.cloud.azure.storage.AzureStorageService;
2524
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
2625
import org.elasticsearch.common.settings.Settings;
@@ -31,77 +30,76 @@
3130
import org.elasticsearch.env.TestEnvironment;
3231
import org.elasticsearch.test.ESTestCase;
3332

34-
import java.io.IOException;
35-
import java.net.URISyntaxException;
36-
3733
import static org.hamcrest.Matchers.is;
34+
import static org.hamcrest.Matchers.nullValue;
3835
import static org.mockito.Mockito.mock;
3936

4037
public class AzureRepositorySettingsTests extends ESTestCase {
4138

42-
private AzureRepository azureRepository(Settings settings) throws StorageException, IOException, URISyntaxException {
39+
private AzureRepository azureRepository(Settings settings) {
4340
Settings internalSettings = Settings.builder()
4441
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath())
4542
.putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths())
4643
.put(settings)
4744
.build();
48-
return new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings),
45+
final AzureRepository azureRepository = new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings),
4946
TestEnvironment.newEnvironment(internalSettings), NamedXContentRegistry.EMPTY, mock(AzureStorageService.class));
47+
assertThat(azureRepository.getBlobStore(), is(nullValue()));
48+
return azureRepository;
5049
}
5150

52-
53-
public void testReadonlyDefault() throws StorageException, IOException, URISyntaxException {
51+
public void testReadonlyDefault() {
5452
assertThat(azureRepository(Settings.EMPTY).isReadOnly(), is(false));
5553
}
5654

57-
public void testReadonlyDefaultAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
55+
public void testReadonlyDefaultAndReadonlyOn() {
5856
assertThat(azureRepository(Settings.builder()
5957
.put("readonly", true)
6058
.build()).isReadOnly(), is(true));
6159
}
6260

63-
public void testReadonlyWithPrimaryOnly() throws StorageException, IOException, URISyntaxException {
61+
public void testReadonlyWithPrimaryOnly() {
6462
assertThat(azureRepository(Settings.builder()
6563
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_ONLY.name())
6664
.build()).isReadOnly(), is(false));
6765
}
6866

69-
public void testReadonlyWithPrimaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
67+
public void testReadonlyWithPrimaryOnlyAndReadonlyOn() {
7068
assertThat(azureRepository(Settings.builder()
7169
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_ONLY.name())
7270
.put("readonly", true)
7371
.build()).isReadOnly(), is(true));
7472
}
7573

76-
public void testReadonlyWithSecondaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
74+
public void testReadonlyWithSecondaryOnlyAndReadonlyOn() {
7775
assertThat(azureRepository(Settings.builder()
7876
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.SECONDARY_ONLY.name())
7977
.put("readonly", true)
8078
.build()).isReadOnly(), is(true));
8179
}
8280

83-
public void testReadonlyWithSecondaryOnlyAndReadonlyOff() throws StorageException, IOException, URISyntaxException {
81+
public void testReadonlyWithSecondaryOnlyAndReadonlyOff() {
8482
assertThat(azureRepository(Settings.builder()
8583
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.SECONDARY_ONLY.name())
8684
.put("readonly", false)
8785
.build()).isReadOnly(), is(false));
8886
}
8987

90-
public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
88+
public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOn() {
9189
assertThat(azureRepository(Settings.builder()
9290
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_THEN_SECONDARY.name())
9391
.put("readonly", true)
9492
.build()).isReadOnly(), is(true));
9593
}
9694

97-
public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOff() throws StorageException, IOException, URISyntaxException {
95+
public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOff() {
9896
assertThat(azureRepository(Settings.builder()
9997
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_THEN_SECONDARY.name())
10098
.put("readonly", false)
10199
.build()).isReadOnly(), is(false));
102100
}
103101

104-
public void testChunkSize() throws StorageException, IOException, URISyntaxException {
102+
public void testChunkSize() {
105103
// default chunk size
106104
AzureRepository azureRepository = azureRepository(Settings.EMPTY);
107105
assertEquals(AzureStorageService.MAX_CHUNK_SIZE, azureRepository.chunkSize());

0 commit comments

Comments
 (0)