Skip to content

Commit 7f271fd

Browse files
beiskedadoonet
beiske
authored and
dadoonet
committed
Add per repository credentials
Changed AwsS3Service to use one client per region and credentials combination. Made S3Repository specify credentials if such exists in the repository settings. Updated readme with repository specific credentials settings. Closes #54. Closes #55. Closes #56. (cherry picked from commit d4ea2dd)
1 parent 254fb81 commit 7f271fd

File tree

5 files changed

+269
-87
lines changed

5 files changed

+269
-87
lines changed

‎README.md

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ The following settings are supported:
119119
* `bucket`: The name of the bucket to be used for snapshots. (Mandatory)
120120
* `region`: The region where bucket is located. Defaults to US Standard
121121
* `base_path`: Specifies the path within bucket to repository data. Defaults to root directory.
122+
* `access_key`: The access key to use for authentication. Defaults to value of `cloud.aws.access_key`.
123+
* `secret_key`: The secret key to use for authentication. Defaults to value of `cloud.aws.secret_key`.
122124
* `concurrent_streams`: Throttles the number of streams (per node) preforming snapshot operation. Defaults to `5`.
123125
* `chunk_size`: Big files can be broken down into chunks during snapshotting if needed. The chunk size can be specified in bytes or by using size value notation, i.e. `1g`, `10m`, `5k`. Defaults to `100m`.
124126
* `compress`: When set to `true` metadata files are stored in compressed format. This setting doesn't affect index files that are already compressed by default. Defaults to `false`.
@@ -131,11 +133,11 @@ The S3 repositories are using the same credentials as the rest of the S3 service
131133
secret_key: vExyMThREXeRMm/b/LRzEB8jWwvzQeXgjqMX+6br
132134

133135

134-
Multiple S3 repositories can be created as long as they share the same credential.
136+
Multiple S3 repositories can be created. If the buckets require different credentials, then define them as part of the repository settings.
135137

136138
## Testing
137139

138-
Integrations tests in this plugin require working AWS configuration and therefore disabled by default. To enable tests prepare a config file elasticsearch.yml with the following content:
140+
Integrations tests in this plugin require working AWS configuration and therefore disabled by default. Three buckets and two iam users have to be created. The first iam user needs access to two buckets in different regions and the final bucket is exclusive for the other iam user. To enable tests prepare a config file elasticsearch.yml with the following content:
139141

140142
```
141143
cloud:
@@ -147,10 +149,17 @@ repositories:
147149
s3:
148150
bucket: "bucket_name"
149151
region: "us-west-2"
152+
private-bucket:
153+
bucket: <bucket not accessible by default key>
154+
access_key: <access key>
155+
secret_key: <access key>
156+
remote-bucket:
157+
bucket: <bucket in other region>
158+
region: <region>
150159
151160
```
152161

153-
Replaces `access_key`, `secret_key`, `bucket` and `region` with your settings. Please, note that the test will delete all snapshot/restore related files in the specified bucket.
162+
Replace all occurrences of `access_key`, `secret_key`, `bucket` and `region` with your settings. Please, note that the test will delete all snapshot/restore related files in the specified buckets.
154163

155164
To run test:
156165

‎src/main/java/org/elasticsearch/cloud/aws/AwsS3Service.java

Lines changed: 83 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919

2020
package org.elasticsearch.cloud.aws;
2121

22+
import java.util.HashMap;
23+
import java.util.Map;
24+
2225
import com.amazonaws.ClientConfiguration;
2326
import com.amazonaws.Protocol;
2427
import com.amazonaws.auth.*;
@@ -27,6 +30,7 @@
2730
import com.amazonaws.services.s3.AmazonS3Client;
2831
import org.elasticsearch.ElasticsearchException;
2932
import org.elasticsearch.ElasticsearchIllegalArgumentException;
33+
import org.elasticsearch.common.collect.Tuple;
3034
import org.elasticsearch.common.component.AbstractLifecycleComponent;
3135
import org.elasticsearch.common.inject.Inject;
3236
import org.elasticsearch.common.settings.Settings;
@@ -37,7 +41,10 @@
3741
*/
3842
public class AwsS3Service extends AbstractLifecycleComponent<AwsS3Service> {
3943

40-
private AmazonS3Client client;
44+
/**
45+
* (acceskey, endpoint) -> client
46+
*/
47+
private Map<Tuple<String, String>, AmazonS3Client> clients = new HashMap<Tuple<String,String>, AmazonS3Client>();
4148

4249
@Inject
4350
public AwsS3Service(Settings settings, SettingsFilter settingsFilter) {
@@ -47,6 +54,33 @@ public AwsS3Service(Settings settings, SettingsFilter settingsFilter) {
4754
}
4855

4956
public synchronized AmazonS3 client() {
57+
String endpoint = getDefaultEndpoint();
58+
String account = componentSettings.get("access_key", settings.get("cloud.account"));
59+
String key = componentSettings.get("secret_key", settings.get("cloud.key"));
60+
61+
return getClient(endpoint, account, key);
62+
}
63+
64+
public synchronized AmazonS3 client(String region, String account, String key) {
65+
String endpoint;
66+
if (region == null) {
67+
endpoint = getDefaultEndpoint();
68+
} else {
69+
endpoint = getEndpoint(region);
70+
logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
71+
}
72+
if (account == null || key == null) {
73+
account = componentSettings.get("access_key", settings.get("cloud.account"));
74+
key = componentSettings.get("secret_key", settings.get("cloud.key"));
75+
}
76+
77+
return getClient(endpoint, account, key);
78+
}
79+
80+
81+
private synchronized AmazonS3 getClient(String endpoint, String account, String key) {
82+
Tuple<String, String> clientDescriptor = new Tuple<String, String>(endpoint, account);
83+
AmazonS3Client client = clients.get(clientDescriptor);
5084
if (client != null) {
5185
return client;
5286
}
@@ -60,8 +94,6 @@ public synchronized AmazonS3 client() {
6094
} else {
6195
throw new ElasticsearchIllegalArgumentException("No protocol supported [" + protocol + "], can either be [http] or [https]");
6296
}
63-
String account = componentSettings.get("access_key", settings.get("cloud.account"));
64-
String key = componentSettings.get("secret_key", settings.get("cloud.key"));
6597

6698
String proxyHost = componentSettings.get("proxy_host");
6799
if (proxyHost != null) {
@@ -88,53 +120,60 @@ public synchronized AmazonS3 client() {
88120
new StaticCredentialsProvider(new BasicAWSCredentials(account, key))
89121
);
90122
}
91-
this.client = new AmazonS3Client(credentials, clientConfiguration);
123+
client = new AmazonS3Client(credentials, clientConfiguration);
92124

125+
if (endpoint != null) {
126+
client.setEndpoint(endpoint);
127+
}
128+
clients.put(clientDescriptor, client);
129+
return client;
130+
}
131+
132+
private String getDefaultEndpoint() {
133+
String endpoint = null;
93134
if (componentSettings.get("s3.endpoint") != null) {
94-
String endpoint = componentSettings.get("s3.endpoint");
135+
endpoint = componentSettings.get("s3.endpoint");
95136
logger.debug("using explicit s3 endpoint [{}]", endpoint);
96-
client.setEndpoint(endpoint);
97137
} else if (componentSettings.get("region") != null) {
98-
String endpoint;
99138
String region = componentSettings.get("region").toLowerCase();
100-
if ("us-east".equals(region)) {
101-
endpoint = "s3.amazonaws.com";
102-
} else if ("us-east-1".equals(region)) {
103-
endpoint = "s3.amazonaws.com";
104-
} else if ("us-west".equals(region)) {
105-
endpoint = "s3-us-west-1.amazonaws.com";
106-
} else if ("us-west-1".equals(region)) {
107-
endpoint = "s3-us-west-1.amazonaws.com";
108-
} else if ("us-west-2".equals(region)) {
109-
endpoint = "s3-us-west-2.amazonaws.com";
110-
} else if ("ap-southeast".equals(region)) {
111-
endpoint = "s3-ap-southeast-1.amazonaws.com";
112-
} else if ("ap-southeast-1".equals(region)) {
113-
endpoint = "s3-ap-southeast-1.amazonaws.com";
114-
} else if ("ap-southeast-2".equals(region)) {
115-
endpoint = "s3-ap-southeast-2.amazonaws.com";
116-
} else if ("ap-northeast".equals(region)) {
117-
endpoint = "s3-ap-northeast-1.amazonaws.com";
118-
} else if ("ap-northeast-1".equals(region)) {
119-
endpoint = "s3-ap-northeast-1.amazonaws.com";
120-
} else if ("eu-west".equals(region)) {
121-
endpoint = "s3-eu-west-1.amazonaws.com";
122-
} else if ("eu-west-1".equals(region)) {
123-
endpoint = "s3-eu-west-1.amazonaws.com";
124-
} else if ("sa-east".equals(region)) {
125-
endpoint = "s3-sa-east-1.amazonaws.com";
126-
} else if ("sa-east-1".equals(region)) {
127-
endpoint = "s3-sa-east-1.amazonaws.com";
128-
} else {
129-
throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
130-
}
131-
if (endpoint != null) {
132-
logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
133-
client.setEndpoint(endpoint);
134-
}
139+
endpoint = getEndpoint(region);
140+
logger.debug("using s3 region [{}], with endpoint [{}]", region, endpoint);
135141
}
142+
return endpoint;
143+
}
136144

137-
return this.client;
145+
private static String getEndpoint(String region) {
146+
if ("us-east".equals(region)) {
147+
return "s3.amazonaws.com";
148+
} else if ("us-east-1".equals(region)) {
149+
return "s3.amazonaws.com";
150+
} else if ("us-west".equals(region)) {
151+
return "s3-us-west-1.amazonaws.com";
152+
} else if ("us-west-1".equals(region)) {
153+
return "s3-us-west-1.amazonaws.com";
154+
} else if ("us-west-2".equals(region)) {
155+
return "s3-us-west-2.amazonaws.com";
156+
} else if ("ap-southeast".equals(region)) {
157+
return "s3-ap-southeast-1.amazonaws.com";
158+
} else if ("ap-southeast-1".equals(region)) {
159+
return "s3-ap-southeast-1.amazonaws.com";
160+
} else if ("ap-southeast-2".equals(region)) {
161+
return "s3-ap-southeast-2.amazonaws.com";
162+
} else if ("ap-northeast".equals(region)) {
163+
return "s3-ap-northeast-1.amazonaws.com";
164+
} else if ("ap-northeast-1".equals(region)) {
165+
return "s3-ap-northeast-1.amazonaws.com";
166+
} else if ("eu-west".equals(region)) {
167+
return "s3-eu-west-1.amazonaws.com";
168+
} else if ("eu-west-1".equals(region)) {
169+
return "s3-eu-west-1.amazonaws.com";
170+
} else if ("sa-east".equals(region)) {
171+
return "s3-sa-east-1.amazonaws.com";
172+
} else if ("sa-east-1".equals(region)) {
173+
return "s3-sa-east-1.amazonaws.com";
174+
} else {
175+
throw new ElasticsearchIllegalArgumentException("No automatic endpoint could be derived from region [" + region + "]");
176+
}
138177
}
139178

140179
@Override
@@ -147,7 +186,7 @@ protected void doStop() throws ElasticsearchException {
147186

148187
@Override
149188
protected void doClose() throws ElasticsearchException {
150-
if (client != null) {
189+
for (AmazonS3Client client : clients.values()) {
151190
client.shutdown();
152191
}
153192
}

‎src/main/java/org/elasticsearch/repositories/s3/S3Repository.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ public S3Repository(RepositoryName name, RepositorySettings repositorySettings,
124124
ExecutorService concurrentStreamPool = EsExecutors.newScaling(1, concurrentStreams, 5, TimeUnit.SECONDS, EsExecutors.daemonThreadFactory(settings, "[s3_stream]"));
125125

126126
logger.debug("using bucket [{}], region [{}], chunk_size [{}], concurrent_streams [{}]", bucket, region, chunkSize, concurrentStreams);
127-
blobStore = new S3BlobStore(settings, s3Service.client(), bucket, region, concurrentStreamPool);
127+
blobStore = new S3BlobStore(settings, s3Service.client(region, repositorySettings.settings().get("access_key"), repositorySettings.settings().get("secret_key")), bucket, region, concurrentStreamPool);
128128
this.chunkSize = repositorySettings.settings().getAsBytesSize("chunk_size", componentSettings.getAsBytesSize("chunk_size", new ByteSizeValue(100, ByteSizeUnit.MB)));
129129
this.compress = repositorySettings.settings().getAsBoolean("compress", componentSettings.getAsBoolean("compress", false));
130130
String basePath = repositorySettings.settings().get("base_path", null);

0 commit comments

Comments
 (0)