Skip to content

Commit 54de108

Browse files
dadoonetdadoonet
committed
Use Azure Storage 2.0.0
Microsoft team has released a new specific project to deal with storage with a much cleaner API than the previous version. See https://github.com/azure/azure-storage-java Documentation is here: http://azure.microsoft.com/en-us/documentation/articles/storage-java-how-to-use-blob-storage/ Note that the produced ZIP file has been reduced from 5mb to 1.3mb. Related to #38 (cherry picked from commit 4467254) (cherry picked from commit b2f1e4d)
1 parent c20371a commit 54de108

File tree

10 files changed

+51
-82
lines changed

10 files changed

+51
-82
lines changed

‎pom.xml

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -68,15 +68,9 @@ governing permissions and limitations under the License. -->
6868

6969
<!-- Azure API -->
7070
<dependency>
71-
<groupId>com.microsoft.windowsazure</groupId>
72-
<artifactId>microsoft-windowsazure-api</artifactId>
73-
<version>0.4.6</version>
74-
<exclusions>
75-
<exclusion>
76-
<groupId>javax.xml.stream</groupId>
77-
<artifactId>stax-api</artifactId>
78-
</exclusion>
79-
</exclusions>
71+
<groupId>com.microsoft.azure</groupId>
72+
<artifactId>azure-storage</artifactId>
73+
<version>2.0.0</version>
8074
</dependency>
8175

8276
<dependency>

‎src/main/assemblies/plugin.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ governing permissions and limitations under the License. -->
3030
<useProjectArtifact>true</useProjectArtifact>
3131
<useTransitiveFiltering>true</useTransitiveFiltering>
3232
<includes>
33-
<include>com.microsoft.windowsazure:microsoft-windowsazure-api</include>
33+
<include>com.microsoft.azure:azure-storage</include>
3434
</includes>
3535
</dependencySet>
3636
</dependencySets>

‎src/main/java/org/elasticsearch/cloud/azure/AzureStorageService.java

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@
1919

2020
package org.elasticsearch.cloud.azure;
2121

22-
import com.microsoft.windowsazure.services.core.ServiceException;
23-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.common.blobstore.BlobMetaData;
2524
import org.elasticsearch.common.collect.ImmutableMap;
2625

@@ -48,17 +47,17 @@ static public final class Fields {
4847

4948
void createContainer(String container) throws URISyntaxException, StorageException;
5049

51-
void deleteFiles(String container, String path) throws URISyntaxException, StorageException, ServiceException;
50+
void deleteFiles(String container, String path) throws URISyntaxException, StorageException;
5251

5352
boolean blobExists(String container, String blob) throws URISyntaxException, StorageException;
5453

5554
void deleteBlob(String container, String blob) throws URISyntaxException, StorageException;
5655

57-
InputStream getInputStream(String container, String blob) throws ServiceException;
56+
InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException;
5857

5958
OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException;
6059

61-
ImmutableMap<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException;
60+
ImmutableMap<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException;
6261

6362
void moveBlob(String container, String sourceBlob, String targetBlob) throws URISyntaxException, StorageException;
6463
}

‎src/main/java/org/elasticsearch/cloud/azure/AzureStorageServiceImpl.java

Lines changed: 21 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,9 @@
1919

2020
package org.elasticsearch.cloud.azure;
2121

22-
import com.microsoft.windowsazure.services.blob.BlobConfiguration;
23-
import com.microsoft.windowsazure.services.blob.BlobContract;
24-
import com.microsoft.windowsazure.services.blob.BlobService;
25-
import com.microsoft.windowsazure.services.blob.client.CloudBlobClient;
26-
import com.microsoft.windowsazure.services.blob.client.CloudBlobContainer;
27-
import com.microsoft.windowsazure.services.blob.client.CloudBlockBlob;
28-
import com.microsoft.windowsazure.services.blob.client.ListBlobItem;
29-
import com.microsoft.windowsazure.services.blob.models.BlobProperties;
30-
import com.microsoft.windowsazure.services.blob.models.GetBlobResult;
31-
import com.microsoft.windowsazure.services.blob.models.ListBlobsOptions;
32-
import com.microsoft.windowsazure.services.blob.models.ListBlobsResult;
33-
import com.microsoft.windowsazure.services.core.Configuration;
34-
import com.microsoft.windowsazure.services.core.ServiceException;
35-
import com.microsoft.windowsazure.services.core.storage.CloudStorageAccount;
36-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.CloudStorageAccount;
23+
import com.microsoft.azure.storage.StorageException;
24+
import com.microsoft.azure.storage.blob.*;
3725
import org.elasticsearch.ElasticsearchException;
3826
import org.elasticsearch.common.blobstore.BlobMetaData;
3927
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
@@ -48,7 +36,6 @@
4836
import java.io.OutputStream;
4937
import java.net.URI;
5038
import java.net.URISyntaxException;
51-
import java.util.List;
5239

5340
/**
5441
*
@@ -60,10 +47,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
6047
private final String key;
6148
private final String blob;
6249

63-
private CloudStorageAccount storage_account;
6450
private CloudBlobClient client;
65-
private BlobContract service;
66-
6751

6852
@Inject
6953
public AzureStorageServiceImpl(Settings settings, SettingsFilter settingsFilter) {
@@ -84,14 +68,11 @@ public AzureStorageServiceImpl(Settings settings, SettingsFilter settingsFilter)
8468
+ "AccountName="+ account +";"
8569
+ "AccountKey=" + key;
8670

87-
Configuration configuration = Configuration.getInstance();
88-
configuration.setProperty(BlobConfiguration.ACCOUNT_NAME, account);
89-
configuration.setProperty(BlobConfiguration.ACCOUNT_KEY, key);
90-
configuration.setProperty(BlobConfiguration.URI, blob);
91-
service = BlobService.create(configuration);
71+
// Retrieve storage account from connection-string.
72+
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
9273

93-
storage_account = CloudStorageAccount.parse(storageConnectionString);
94-
client = storage_account.createCloudBlobClient();
74+
// Create the blob client.
75+
client = storageAccount.createCloudBlobClient();
9576
}
9677
} catch (Exception e) {
9778
// Can not start Azure Storage Client
@@ -129,27 +110,23 @@ public void createContainer(String container) throws URISyntaxException, Storage
129110
try {
130111
CloudBlobContainer blob_container = client.getContainerReference(container);
131112
logger.trace("creating container [{}]", container);
132-
blob_container.createIfNotExist();
113+
blob_container.createIfNotExists();
133114
} catch (IllegalArgumentException e) {
134115
logger.trace("fails creating container [{}]", container, e.getMessage());
135116
throw new RepositoryException(container, e.getMessage());
136117
}
137118
}
138119

139120
@Override
140-
public void deleteFiles(String container, String path) throws URISyntaxException, StorageException, ServiceException {
121+
public void deleteFiles(String container, String path) throws URISyntaxException, StorageException {
141122
logger.trace("delete files container [{}], path [{}]", container, path);
142123

143124
// Container name must be lower case.
144125
CloudBlobContainer blob_container = client.getContainerReference(container);
145126
if (blob_container.exists()) {
146-
ListBlobsOptions options = new ListBlobsOptions();
147-
options.setPrefix(path);
148-
149-
List<ListBlobsResult.BlobEntry> blobs = service.listBlobs(container, options).getBlobs();
150-
for (ListBlobsResult.BlobEntry blob : blobs) {
151-
logger.trace("removing in container [{}], path [{}], blob [{}]", container, path, blob.getName());
152-
service.deleteBlob(container, blob.getName());
127+
for (ListBlobItem blobItem : blob_container.listBlobs(path)) {
128+
logger.trace("removing blob [{}]", blobItem.getUri());
129+
deleteBlob(container, blobItem.getUri().toString());
153130
}
154131
}
155132
}
@@ -180,10 +157,9 @@ public void deleteBlob(String container, String blob) throws URISyntaxException,
180157
}
181158

182159
@Override
183-
public InputStream getInputStream(String container, String blob) throws ServiceException {
160+
public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException {
184161
logger.trace("reading container [{}], blob [{}]", container, blob);
185-
GetBlobResult blobResult = service.getBlob(container, blob);
186-
return blobResult.getContentStream();
162+
return client.getContainerReference(container).getBlockBlobReference(blob).openInputStream();
187163
}
188164

189165
@Override
@@ -193,21 +169,20 @@ public OutputStream getOutputStream(String container, String blob) throws URISyn
193169
}
194170

195171
@Override
196-
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException {
172+
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException {
197173
logger.debug("listing container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix);
198174
ImmutableMap.Builder<String, BlobMetaData> blobsBuilder = ImmutableMap.builder();
199175

200176
CloudBlobContainer blob_container = client.getContainerReference(container);
201177
if (blob_container.exists()) {
202-
Iterable<ListBlobItem> blobs = blob_container.listBlobs(keyPath + prefix);
203-
for (ListBlobItem blob : blobs) {
204-
URI uri = blob.getUri();
178+
for (ListBlobItem blobItem : blob_container.listBlobs(keyPath + prefix)) {
179+
URI uri = blobItem.getUri();
205180
logger.trace("blob url [{}]", uri);
206181
String blobpath = uri.getPath().substring(container.length() + 1);
207-
BlobProperties properties = service.getBlobProperties(container, blobpath).getProperties();
182+
BlobProperties properties = blob_container.getBlockBlobReference(blobpath).getProperties();
208183
String name = blobpath.substring(keyPath.length() + 1);
209-
logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getContentLength());
210-
blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getContentLength()));
184+
logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength());
185+
blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
211186
}
212187
}
213188

@@ -221,7 +196,7 @@ public void moveBlob(String container, String sourceBlob, String targetBlob) thr
221196
CloudBlockBlob blobSource = blob_container.getBlockBlobReference(sourceBlob);
222197
if (blobSource.exists()) {
223198
CloudBlockBlob blobTarget = blob_container.getBlockBlobReference(targetBlob);
224-
blobTarget.copyFromBlob(blobSource);
199+
blobTarget.startCopyFromBlob(blobSource);
225200
blobSource.delete();
226201
logger.debug("moveBlob container [{}], sourceBlob [{}], targetBlob [{}] -> done", container, sourceBlob, targetBlob);
227202
}

‎src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@
1919

2020
package org.elasticsearch.cloud.azure.blobstore;
2121

22-
import com.microsoft.windowsazure.services.core.ServiceException;
23-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.common.Nullable;
2524
import org.elasticsearch.common.blobstore.BlobMetaData;
2625
import org.elasticsearch.common.blobstore.BlobPath;
@@ -73,11 +72,13 @@ public boolean blobExists(String blobName) {
7372
public InputStream openInput(String blobName) throws IOException {
7473
try {
7574
return blobStore.client().getInputStream(blobStore.container(), buildKey(blobName));
76-
} catch (ServiceException e) {
75+
} catch (StorageException e) {
7776
if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
7877
throw new FileNotFoundException(e.getMessage());
7978
}
8079
throw new IOException(e);
80+
} catch (URISyntaxException e) {
81+
throw new IOException(e);
8182
}
8283
}
8384

@@ -112,7 +113,7 @@ public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(@Nullable String pre
112113

113114
try {
114115
return blobStore.client().listBlobsByPrefix(blobStore.container(), keyPath, prefix);
115-
} catch (URISyntaxException | StorageException | ServiceException e) {
116+
} catch (URISyntaxException | StorageException e) {
116117
logger.warn("can not access [{}] in container {{}}: {}", prefix, blobStore.container(), e.getMessage());
117118
throw new IOException(e);
118119
}

‎src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@
1919

2020
package org.elasticsearch.cloud.azure.blobstore;
2121

22-
import com.microsoft.windowsazure.services.core.ServiceException;
23-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.cloud.azure.AzureStorageService;
2524
import org.elasticsearch.common.blobstore.BlobContainer;
2625
import org.elasticsearch.common.blobstore.BlobPath;
@@ -81,7 +80,7 @@ public void delete(BlobPath path) {
8180

8281
try {
8382
client.deleteFiles(container, keyPath);
84-
} catch (URISyntaxException | StorageException | ServiceException e) {
83+
} catch (URISyntaxException | StorageException e) {
8584
logger.warn("can not remove [{}] in container {{}}: {}", keyPath, container, e.getMessage());
8685
}
8786
}

‎src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
package org.elasticsearch.repositories.azure;
2121

22-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2323
import org.elasticsearch.cloud.azure.AzureStorageService;
2424
import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
2525
import org.elasticsearch.cluster.metadata.MetaData;

‎src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@
1919

2020
package org.elasticsearch.repositories.azure;
2121

22-
import com.microsoft.windowsazure.services.core.ServiceException;
23-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.cloud.azure.AbstractAzureTest;
2524
import org.elasticsearch.cloud.azure.AzureStorageService;
2625
import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -87,15 +86,15 @@ public Settings indexSettings() {
8786
}
8887

8988
@Before @After
90-
public final void wipe() throws StorageException, ServiceException, URISyntaxException {
89+
public final void wipe() throws StorageException, URISyntaxException {
9190
wipeRepositories();
9291
cleanRepositoryFiles(basePath);
9392
}
9493

9594
/**
9695
* Purge the test container
9796
*/
98-
public void cleanRepositoryFiles(String path) throws StorageException, ServiceException, URISyntaxException {
97+
public void cleanRepositoryFiles(String path) throws StorageException, URISyntaxException {
9998
String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container");
10099
logger.info("--> remove blobs in container [{}]", container);
101100
AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);

‎src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@
2020
package org.elasticsearch.repositories.azure;
2121

2222

23-
import com.microsoft.windowsazure.services.core.ServiceException;
24-
import com.microsoft.windowsazure.services.core.storage.StorageException;
23+
import com.microsoft.azure.storage.StorageException;
2524
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
2625
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
2726
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
@@ -83,7 +82,7 @@ public Settings indexSettings() {
8382
}
8483

8584
@Before
86-
public final void wipeBefore() throws StorageException, ServiceException, URISyntaxException {
85+
public final void wipeBefore() throws StorageException, URISyntaxException {
8786
wipeRepositories();
8887
cleanRepositoryFiles(
8988
getContainerName(),
@@ -92,7 +91,7 @@ public final void wipeBefore() throws StorageException, ServiceException, URISyn
9291
}
9392

9493
@After
95-
public final void wipeAfter() throws StorageException, ServiceException, URISyntaxException {
94+
public final void wipeAfter() throws StorageException, URISyntaxException {
9695
wipeRepositories();
9796
cleanRepositoryFiles(
9897
getContainerName(),
@@ -243,7 +242,7 @@ public void testMultipleRepositories() {
243242
* For issue #26: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/26
244243
*/
245244
@Test
246-
public void testListBlobs_26() throws StorageException, ServiceException, URISyntaxException {
245+
public void testListBlobs_26() throws StorageException, URISyntaxException {
247246
createIndex("test-idx-1", "test-idx-2", "test-idx-3");
248247
ensureGreen();
249248

@@ -302,7 +301,7 @@ public void testListBlobs_26() throws StorageException, ServiceException, URISyn
302301
* For issue #28: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/28
303302
*/
304303
@Test
305-
public void testGetDeleteNonExistingSnapshot_28() throws StorageException, ServiceException, URISyntaxException {
304+
public void testGetDeleteNonExistingSnapshot_28() throws StorageException, URISyntaxException {
306305
ClusterAdminClient client = client().admin().cluster();
307306
logger.info("--> creating azure repository without any path");
308307
PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
@@ -452,7 +451,7 @@ public static void wipeRepositories(String... repositories) {
452451
/**
453452
* Purge the test containers
454453
*/
455-
public void cleanRepositoryFiles(String... containers) throws StorageException, ServiceException, URISyntaxException {
454+
public void cleanRepositoryFiles(String... containers) throws StorageException, URISyntaxException {
456455
AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);
457456
for (String container : containers) {
458457
logger.info("--> remove container [{}]", container);

‎src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
package org.elasticsearch.repositories.azure;
2121

22-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2323
import org.elasticsearch.ElasticsearchException;
2424
import org.elasticsearch.cloud.azure.AzureStorageService;
2525
import org.elasticsearch.common.blobstore.BlobMetaData;
@@ -29,7 +29,10 @@
2929
import org.elasticsearch.common.inject.Inject;
3030
import org.elasticsearch.common.settings.Settings;
3131

32-
import java.io.*;
32+
import java.io.ByteArrayInputStream;
33+
import java.io.ByteArrayOutputStream;
34+
import java.io.InputStream;
35+
import java.io.OutputStream;
3336
import java.net.URISyntaxException;
3437
import java.util.Locale;
3538
import java.util.Map;

0 commit comments

Comments
 (0)