Skip to content

Commit b1bf643

Browse files
vladimirdolzhenkovladimirdolzhenko
authored
lazy snapshot repository initialization (#31606)
lazy snapshot repository initialization
1 parent 0edb096 commit b1bf643

File tree

23 files changed

+539
-216
lines changed

23 files changed

+539
-216
lines changed

‎modules/repository-url/src/main/java/org/elasticsearch/repositories/url/URLRepository.java

+21-10
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
package org.elasticsearch.repositories.url;
2121

2222
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
23+
import org.elasticsearch.common.blobstore.BlobContainer;
2324
import org.elasticsearch.common.blobstore.BlobPath;
2425
import org.elasticsearch.common.blobstore.BlobStore;
2526
import org.elasticsearch.common.blobstore.url.URLBlobStore;
@@ -31,7 +32,6 @@
3132
import org.elasticsearch.repositories.RepositoryException;
3233
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
3334

34-
import java.io.IOException;
3535
import java.net.MalformedURLException;
3636
import java.net.URISyntaxException;
3737
import java.net.URL;
@@ -71,33 +71,44 @@ public class URLRepository extends BlobStoreRepository {
7171

7272
private final Environment environment;
7373

74-
private final URLBlobStore blobStore;
75-
7674
private final BlobPath basePath;
7775

76+
private final URL url;
77+
7878
/**
7979
* Constructs a read-only URL-based repository
8080
*/
8181
public URLRepository(RepositoryMetaData metadata, Environment environment,
82-
NamedXContentRegistry namedXContentRegistry) throws IOException {
82+
NamedXContentRegistry namedXContentRegistry) {
8383
super(metadata, environment.settings(), namedXContentRegistry);
8484

8585
if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) == false) {
8686
throw new RepositoryException(metadata.name(), "missing url");
8787
}
88+
this.environment = environment;
8889
supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(settings);
8990
urlWhiteList = ALLOWED_URLS_SETTING.get(settings).toArray(new URIPattern[]{});
90-
this.environment = environment;
91+
basePath = BlobPath.cleanPath();
92+
url = URL_SETTING.exists(metadata.settings())
93+
? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
94+
}
9195

92-
URL url = URL_SETTING.exists(metadata.settings()) ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
96+
@Override
97+
protected BlobStore createBlobStore() {
9398
URL normalizedURL = checkURL(url);
94-
blobStore = new URLBlobStore(settings, normalizedURL);
95-
basePath = BlobPath.cleanPath();
99+
return new URLBlobStore(settings, normalizedURL);
100+
}
101+
102+
// only use for testing
103+
@Override
104+
protected BlobContainer blobContainer() {
105+
return super.blobContainer();
96106
}
97107

108+
// only use for testing
98109
@Override
99-
protected BlobStore blobStore() {
100-
return blobStore;
110+
protected BlobStore getBlobStore() {
111+
return super.getBlobStore();
101112
}
102113

103114
@Override

‎modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLRepositoryTests.java

+45-6
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,22 @@
3131
import java.nio.file.Path;
3232
import java.util.Collections;
3333

34+
import static org.hamcrest.CoreMatchers.is;
35+
import static org.hamcrest.CoreMatchers.not;
36+
import static org.hamcrest.CoreMatchers.nullValue;
37+
3438
public class URLRepositoryTests extends ESTestCase {
3539

40+
private URLRepository createRepository(Settings baseSettings, RepositoryMetaData repositoryMetaData) {
41+
return new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
42+
new NamedXContentRegistry(Collections.emptyList())) {
43+
@Override
44+
protected void assertSnapshotOrGenericThread() {
45+
// eliminate thread name check as we create repo manually on test/main threads
46+
}
47+
};
48+
}
49+
3650
public void testWhiteListingRepoURL() throws IOException {
3751
String repoPath = createTempDir().resolve("repository").toUri().toURL().toString();
3852
Settings baseSettings = Settings.builder()
@@ -41,8 +55,12 @@ public void testWhiteListingRepoURL() throws IOException {
4155
.put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
4256
.build();
4357
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
44-
new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
45-
new NamedXContentRegistry(Collections.emptyList()));
58+
final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
59+
repository.start();
60+
61+
assertThat("blob store has to be lazy initialized", repository.getBlobStore(), is(nullValue()));
62+
repository.blobContainer();
63+
assertThat("blobContainer has to initialize blob store", repository.getBlobStore(), not(nullValue()));
4664
}
4765

4866
public void testIfNotWhiteListedMustSetRepoURL() throws IOException {
@@ -52,9 +70,10 @@ public void testIfNotWhiteListedMustSetRepoURL() throws IOException {
5270
.put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), repoPath)
5371
.build();
5472
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
73+
final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
74+
repository.start();
5575
try {
56-
new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
57-
new NamedXContentRegistry(Collections.emptyList()));
76+
repository.blobContainer();
5877
fail("RepositoryException should have been thrown.");
5978
} catch (RepositoryException e) {
6079
String msg = "[url] file url [" + repoPath
@@ -73,13 +92,33 @@ public void testMustBeSupportedProtocol() throws IOException {
7392
.put(URLRepository.SUPPORTED_PROTOCOLS_SETTING.getKey(), "http,https")
7493
.build();
7594
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
95+
final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
96+
repository.start();
7697
try {
77-
new URLRepository(repositoryMetaData, TestEnvironment.newEnvironment(baseSettings),
78-
new NamedXContentRegistry(Collections.emptyList()));
98+
repository.blobContainer();
7999
fail("RepositoryException should have been thrown.");
80100
} catch (RepositoryException e) {
81101
assertEquals("[url] unsupported url protocol [file] from URL [" + repoPath +"]", e.getMessage());
82102
}
83103
}
84104

105+
public void testNonNormalizedUrl() throws IOException {
106+
Settings baseSettings = Settings.builder()
107+
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
108+
.put(URLRepository.ALLOWED_URLS_SETTING.getKey(), "file:/tmp/")
109+
.put(URLRepository.REPOSITORIES_URL_SETTING.getKey(), "file:/var/" )
110+
.build();
111+
RepositoryMetaData repositoryMetaData = new RepositoryMetaData("url", URLRepository.TYPE, baseSettings);
112+
final URLRepository repository = createRepository(baseSettings, repositoryMetaData);
113+
repository.start();
114+
try {
115+
repository.blobContainer();
116+
fail("RepositoryException should have been thrown.");
117+
} catch (RepositoryException e) {
118+
assertEquals("[url] file url [file:/var/] doesn't match any of the locations "
119+
+ "specified by path.repo or repositories.url.allowed_urls",
120+
e.getMessage());
121+
}
122+
}
123+
85124
}

‎plugins/repository-azure/src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

+28-14
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@
3838
import org.elasticsearch.snapshots.SnapshotCreationException;
3939
import org.elasticsearch.snapshots.SnapshotId;
4040

41-
import java.io.IOException;
4241
import java.net.URISyntaxException;
4342
import java.util.List;
4443
import java.util.Locale;
@@ -78,25 +77,21 @@ public static final class Repository {
7877
public static final Setting<Boolean> READONLY_SETTING = Setting.boolSetting("readonly", false, Property.NodeScope);
7978
}
8079

81-
private final AzureBlobStore blobStore;
8280
private final BlobPath basePath;
8381
private final ByteSizeValue chunkSize;
8482
private final boolean compress;
83+
private final Environment environment;
84+
private final AzureStorageService storageService;
8585
private final boolean readonly;
8686

8787
public AzureRepository(RepositoryMetaData metadata, Environment environment, NamedXContentRegistry namedXContentRegistry,
88-
AzureStorageService storageService) throws IOException, URISyntaxException, StorageException {
88+
AzureStorageService storageService) {
8989
super(metadata, environment.settings(), namedXContentRegistry);
90-
this.blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
9190
this.chunkSize = Repository.CHUNK_SIZE_SETTING.get(metadata.settings());
9291
this.compress = Repository.COMPRESS_SETTING.get(metadata.settings());
93-
// If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
94-
// For secondary_only setting, the repository should be read only
95-
if (Repository.READONLY_SETTING.exists(metadata.settings())) {
96-
this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
97-
} else {
98-
this.readonly = this.blobStore.getLocationMode() == LocationMode.SECONDARY_ONLY;
99-
}
92+
this.environment = environment;
93+
this.storageService = storageService;
94+
10095
final String basePath = Strings.trimLeadingCharacter(Repository.BASE_PATH_SETTING.get(metadata.settings()), '/');
10196
if (Strings.hasLength(basePath)) {
10297
// Remove starting / if any
@@ -108,15 +103,33 @@ public AzureRepository(RepositoryMetaData metadata, Environment environment, Nam
108103
} else {
109104
this.basePath = BlobPath.cleanPath();
110105
}
111-
logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
112-
"using container [{}], chunk_size [{}], compress [{}], base_path [{}]", blobStore, chunkSize, compress, basePath));
106+
107+
// If the user explicitly did not define a readonly value, we set it by ourselves depending on the location mode setting.
108+
// For secondary_only setting, the repository should be read only
109+
final LocationMode locationMode = Repository.LOCATION_MODE_SETTING.get(metadata.settings());
110+
if (Repository.READONLY_SETTING.exists(metadata.settings())) {
111+
this.readonly = Repository.READONLY_SETTING.get(metadata.settings());
112+
} else {
113+
this.readonly = locationMode == LocationMode.SECONDARY_ONLY;
114+
}
115+
}
116+
117+
// only use for testing
118+
@Override
119+
protected BlobStore getBlobStore() {
120+
return super.getBlobStore();
113121
}
114122

115123
/**
116124
* {@inheritDoc}
117125
*/
118126
@Override
119-
protected BlobStore blobStore() {
127+
protected AzureBlobStore createBlobStore() throws URISyntaxException, StorageException {
128+
final AzureBlobStore blobStore = new AzureBlobStore(metadata, environment.settings(), storageService);
129+
130+
logger.debug((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage(
131+
"using container [{}], chunk_size [{}], compress [{}], base_path [{}]",
132+
blobStore, chunkSize, compress, basePath));
120133
return blobStore;
121134
}
122135

@@ -144,6 +157,7 @@ protected ByteSizeValue chunkSize() {
144157
@Override
145158
public void initializeSnapshot(SnapshotId snapshotId, List<IndexId> indices, MetaData clusterMetadata) {
146159
try {
160+
final AzureBlobStore blobStore = (AzureBlobStore) blobStore();
147161
if (blobStore.containerExist() == false) {
148162
throw new IllegalArgumentException("The bucket [" + blobStore + "] does not exist. Please create it before "
149163
+ " creating an azure snapshot repository backed by it.");

‎plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureRepositorySettingsTests.java

+14-15
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
package org.elasticsearch.repositories.azure;
2121

2222
import com.microsoft.azure.storage.LocationMode;
23-
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
2524
import org.elasticsearch.common.settings.Settings;
2625
import org.elasticsearch.common.unit.ByteSizeUnit;
@@ -30,76 +29,76 @@
3029
import org.elasticsearch.env.TestEnvironment;
3130
import org.elasticsearch.test.ESTestCase;
3231

33-
import java.io.IOException;
34-
import java.net.URISyntaxException;
35-
3632
import static org.hamcrest.Matchers.is;
33+
import static org.hamcrest.Matchers.nullValue;
3734
import static org.mockito.Mockito.mock;
3835

3936
public class AzureRepositorySettingsTests extends ESTestCase {
4037

41-
private AzureRepository azureRepository(Settings settings) throws StorageException, IOException, URISyntaxException {
38+
private AzureRepository azureRepository(Settings settings) {
4239
Settings internalSettings = Settings.builder()
4340
.put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath())
4441
.putList(Environment.PATH_DATA_SETTING.getKey(), tmpPaths())
4542
.put(settings)
4643
.build();
47-
return new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings),
44+
final AzureRepository azureRepository = new AzureRepository(new RepositoryMetaData("foo", "azure", internalSettings),
4845
TestEnvironment.newEnvironment(internalSettings), NamedXContentRegistry.EMPTY, mock(AzureStorageService.class));
46+
assertThat(azureRepository.getBlobStore(), is(nullValue()));
47+
return azureRepository;
4948
}
5049

51-
public void testReadonlyDefault() throws StorageException, IOException, URISyntaxException {
50+
public void testReadonlyDefault() {
5251
assertThat(azureRepository(Settings.EMPTY).isReadOnly(), is(false));
5352
}
5453

55-
public void testReadonlyDefaultAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
54+
public void testReadonlyDefaultAndReadonlyOn() {
5655
assertThat(azureRepository(Settings.builder()
5756
.put("readonly", true)
5857
.build()).isReadOnly(), is(true));
5958
}
6059

61-
public void testReadonlyWithPrimaryOnly() throws StorageException, IOException, URISyntaxException {
60+
public void testReadonlyWithPrimaryOnly() {
6261
assertThat(azureRepository(Settings.builder()
6362
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_ONLY.name())
6463
.build()).isReadOnly(), is(false));
6564
}
6665

67-
public void testReadonlyWithPrimaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
66+
public void testReadonlyWithPrimaryOnlyAndReadonlyOn() {
6867
assertThat(azureRepository(Settings.builder()
6968
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_ONLY.name())
7069
.put("readonly", true)
7170
.build()).isReadOnly(), is(true));
7271
}
7372

74-
public void testReadonlyWithSecondaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
73+
public void testReadonlyWithSecondaryOnlyAndReadonlyOn() {
7574
assertThat(azureRepository(Settings.builder()
7675
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.SECONDARY_ONLY.name())
7776
.put("readonly", true)
7877
.build()).isReadOnly(), is(true));
7978
}
8079

81-
public void testReadonlyWithSecondaryOnlyAndReadonlyOff() throws StorageException, IOException, URISyntaxException {
80+
public void testReadonlyWithSecondaryOnlyAndReadonlyOff() {
8281
assertThat(azureRepository(Settings.builder()
8382
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.SECONDARY_ONLY.name())
8483
.put("readonly", false)
8584
.build()).isReadOnly(), is(false));
8685
}
8786

88-
public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOn() throws StorageException, IOException, URISyntaxException {
87+
public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOn() {
8988
assertThat(azureRepository(Settings.builder()
9089
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_THEN_SECONDARY.name())
9190
.put("readonly", true)
9291
.build()).isReadOnly(), is(true));
9392
}
9493

95-
public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOff() throws StorageException, IOException, URISyntaxException {
94+
public void testReadonlyWithPrimaryAndSecondaryOnlyAndReadonlyOff() {
9695
assertThat(azureRepository(Settings.builder()
9796
.put(AzureRepository.Repository.LOCATION_MODE_SETTING.getKey(), LocationMode.PRIMARY_THEN_SECONDARY.name())
9897
.put("readonly", false)
9998
.build()).isReadOnly(), is(false));
10099
}
101100

102-
public void testChunkSize() throws StorageException, IOException, URISyntaxException {
101+
public void testChunkSize() {
103102
// default chunk size
104103
AzureRepository azureRepository = azureRepository(Settings.EMPTY);
105104
assertEquals(AzureStorageService.MAX_CHUNK_SIZE, azureRepository.chunkSize());

‎plugins/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageRepository.java

+11-13
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
2323
import org.elasticsearch.common.Strings;
2424
import org.elasticsearch.common.blobstore.BlobPath;
25-
import org.elasticsearch.common.blobstore.BlobStore;
2625
import org.elasticsearch.common.settings.Setting;
2726
import org.elasticsearch.common.unit.ByteSizeUnit;
2827
import org.elasticsearch.common.unit.ByteSizeValue;
@@ -56,18 +55,19 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
5655
byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope, Property.Dynamic);
5756
static final Setting<String> CLIENT_NAME = new Setting<>("client", "default", Function.identity());
5857

59-
private final ByteSizeValue chunkSize;
60-
private final boolean compress;
58+
private final GoogleCloudStorageService storageService;
6159
private final BlobPath basePath;
62-
private final GoogleCloudStorageBlobStore blobStore;
60+
private final boolean compress;
61+
private final ByteSizeValue chunkSize;
62+
private final String bucket;
63+
private final String clientName;
6364

6465
GoogleCloudStorageRepository(RepositoryMetaData metadata, Environment environment,
6566
NamedXContentRegistry namedXContentRegistry,
66-
GoogleCloudStorageService storageService) throws Exception {
67+
GoogleCloudStorageService storageService) {
6768
super(metadata, environment.settings(), namedXContentRegistry);
69+
this.storageService = storageService;
6870

69-
String bucket = getSetting(BUCKET, metadata);
70-
String clientName = CLIENT_NAME.get(metadata.settings());
7171
String basePath = BASE_PATH.get(metadata.settings());
7272
if (Strings.hasLength(basePath)) {
7373
BlobPath path = new BlobPath();
@@ -81,16 +81,14 @@ class GoogleCloudStorageRepository extends BlobStoreRepository {
8181

8282
this.compress = getSetting(COMPRESS, metadata);
8383
this.chunkSize = getSetting(CHUNK_SIZE, metadata);
84-
84+
this.bucket = getSetting(BUCKET, metadata);
85+
this.clientName = CLIENT_NAME.get(metadata.settings());
8586
logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}]", bucket, basePath, chunkSize, compress);
86-
87-
this.blobStore = new GoogleCloudStorageBlobStore(settings, bucket, clientName, storageService);
8887
}
8988

90-
9189
@Override
92-
protected BlobStore blobStore() {
93-
return blobStore;
90+
protected GoogleCloudStorageBlobStore createBlobStore() {
91+
return new GoogleCloudStorageBlobStore(settings, bucket, clientName, storageService);
9492
}
9593

9694
@Override

0 commit comments

Comments
 (0)