Skip to content

Commit 8859c5f

Browse files
dadoonetdadoonet
committed
Use Azure Storage 2.0.0
Microsoft team has released a new specific project to deal with storage with a much cleaner API than the previous version. See https://github.com/azure/azure-storage-java Documentation is here: http://azure.microsoft.com/en-us/documentation/articles/storage-java-how-to-use-blob-storage/ Note that the produced ZIP file has been reduced from 5mb to 1.3mb. Related to elastic#38
1 parent 68efe93 commit 8859c5f

File tree

10 files changed

+52
-81
lines changed

10 files changed

+52
-81
lines changed

‎pom.xml

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -84,17 +84,13 @@ governing permissions and limitations under the License. -->
8484
<artifactId>elasticsearch</artifactId>
8585
<version>${elasticsearch.version}</version>
8686
</dependency>
87+
8788
<dependency>
88-
<groupId>com.microsoft.windowsazure</groupId>
89-
<artifactId>microsoft-windowsazure-api</artifactId>
90-
<version>0.4.6</version>
91-
<exclusions>
92-
<exclusion>
93-
<groupId>javax.xml.stream</groupId>
94-
<artifactId>stax-api</artifactId>
95-
</exclusion>
96-
</exclusions>
89+
<groupId>com.microsoft.azure</groupId>
90+
<artifactId>azure-storage</artifactId>
91+
<version>2.0.0</version>
9792
</dependency>
93+
9894
<dependency>
9995
<groupId>log4j</groupId>
10096
<artifactId>log4j</artifactId>

‎src/main/assemblies/plugin.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ governing permissions and limitations under the License. -->
3030
<useProjectArtifact>true</useProjectArtifact>
3131
<useTransitiveFiltering>true</useTransitiveFiltering>
3232
<includes>
33-
<include>com.microsoft.windowsazure:microsoft-windowsazure-api</include>
33+
<include>com.microsoft.azure:azure-storage</include>
3434
</includes>
3535
</dependencySet>
3636
</dependencySets>

‎src/main/java/org/elasticsearch/cloud/azure/AzureStorageService.java

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@
1919

2020
package org.elasticsearch.cloud.azure;
2121

22-
import com.microsoft.windowsazure.services.core.ServiceException;
23-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.common.blobstore.BlobMetaData;
2524
import org.elasticsearch.common.collect.ImmutableMap;
2625

@@ -48,15 +47,15 @@ static public final class Fields {
4847

4948
void createContainer(String container) throws URISyntaxException, StorageException;
5049

51-
void deleteFiles(String container, String path) throws URISyntaxException, StorageException, ServiceException;
50+
void deleteFiles(String container, String path) throws URISyntaxException, StorageException;
5251

5352
boolean blobExists(String container, String blob) throws URISyntaxException, StorageException;
5453

5554
void deleteBlob(String container, String blob) throws URISyntaxException, StorageException;
5655

57-
InputStream getInputStream(String container, String blob) throws ServiceException;
56+
InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException;
5857

5958
OutputStream getOutputStream(String container, String blob) throws URISyntaxException, StorageException;
6059

61-
ImmutableMap<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException;
60+
ImmutableMap<String,BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException;
6261
}

‎src/main/java/org/elasticsearch/cloud/azure/AzureStorageServiceImpl.java

Lines changed: 20 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,9 @@
1919

2020
package org.elasticsearch.cloud.azure;
2121

22-
import com.microsoft.windowsazure.services.blob.BlobConfiguration;
23-
import com.microsoft.windowsazure.services.blob.BlobContract;
24-
import com.microsoft.windowsazure.services.blob.BlobService;
25-
import com.microsoft.windowsazure.services.blob.client.CloudBlobClient;
26-
import com.microsoft.windowsazure.services.blob.client.CloudBlobContainer;
27-
import com.microsoft.windowsazure.services.blob.client.CloudBlockBlob;
28-
import com.microsoft.windowsazure.services.blob.client.ListBlobItem;
29-
import com.microsoft.windowsazure.services.blob.models.BlobProperties;
30-
import com.microsoft.windowsazure.services.blob.models.GetBlobResult;
31-
import com.microsoft.windowsazure.services.blob.models.ListBlobsOptions;
32-
import com.microsoft.windowsazure.services.blob.models.ListBlobsResult;
33-
import com.microsoft.windowsazure.services.core.Configuration;
34-
import com.microsoft.windowsazure.services.core.ServiceException;
35-
import com.microsoft.windowsazure.services.core.storage.CloudStorageAccount;
36-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.CloudStorageAccount;
23+
import com.microsoft.azure.storage.StorageException;
24+
import com.microsoft.azure.storage.blob.*;
3725
import org.elasticsearch.ElasticsearchException;
3826
import org.elasticsearch.common.blobstore.BlobMetaData;
3927
import org.elasticsearch.common.blobstore.support.PlainBlobMetaData;
@@ -47,7 +35,6 @@
4735
import java.io.OutputStream;
4836
import java.net.URI;
4937
import java.net.URISyntaxException;
50-
import java.util.List;
5138

5239
/**
5340
*
@@ -59,10 +46,7 @@ public class AzureStorageServiceImpl extends AbstractLifecycleComponent<AzureSto
5946
private final String key;
6047
private final String blob;
6148

62-
private CloudStorageAccount storage_account;
6349
private CloudBlobClient client;
64-
private BlobContract service;
65-
6650

6751
@Inject
6852
public AzureStorageServiceImpl(Settings settings, SettingsFilter settingsFilter) {
@@ -83,14 +67,11 @@ public AzureStorageServiceImpl(Settings settings, SettingsFilter settingsFilter)
8367
+ "AccountName="+ account +";"
8468
+ "AccountKey=" + key;
8569

86-
Configuration configuration = Configuration.getInstance();
87-
configuration.setProperty(BlobConfiguration.ACCOUNT_NAME, account);
88-
configuration.setProperty(BlobConfiguration.ACCOUNT_KEY, key);
89-
configuration.setProperty(BlobConfiguration.URI, blob);
90-
service = BlobService.create(configuration);
70+
// Retrieve storage account from connection-string.
71+
CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString);
9172

92-
storage_account = CloudStorageAccount.parse(storageConnectionString);
93-
client = storage_account.createCloudBlobClient();
73+
// Create the blob client.
74+
client = storageAccount.createCloudBlobClient();
9475
}
9576
} catch (Exception e) {
9677
// Can not start Azure Storage Client
@@ -127,23 +108,19 @@ public void removeContainer(String container) throws URISyntaxException, Storage
127108
public void createContainer(String container) throws URISyntaxException, StorageException {
128109
CloudBlobContainer blob_container = client.getContainerReference(container);
129110
logger.trace("creating container [{}]", container);
130-
blob_container.createIfNotExist();
111+
blob_container.createIfNotExists();
131112
}
132113

133114
@Override
134-
public void deleteFiles(String container, String path) throws URISyntaxException, StorageException, ServiceException {
115+
public void deleteFiles(String container, String path) throws URISyntaxException, StorageException {
135116
logger.trace("delete files container [{}], path [{}]", container, path);
136117

137118
// Container name must be lower case.
138119
CloudBlobContainer blob_container = client.getContainerReference(container);
139120
if (blob_container.exists()) {
140-
ListBlobsOptions options = new ListBlobsOptions();
141-
options.setPrefix(path);
142-
143-
List<ListBlobsResult.BlobEntry> blobs = service.listBlobs(container, options).getBlobs();
144-
for (ListBlobsResult.BlobEntry blob : blobs) {
145-
logger.trace("removing in container [{}], path [{}], blob [{}]", container, path, blob.getName());
146-
service.deleteBlob(container, blob.getName());
121+
for (ListBlobItem blobItem : blob_container.listBlobs(path)) {
122+
logger.trace("removing blob [{}]", blobItem.getUri());
123+
deleteBlob(container, blobItem.getUri().toString());
147124
}
148125
}
149126
}
@@ -174,9 +151,8 @@ public void deleteBlob(String container, String blob) throws URISyntaxException,
174151
}
175152

176153
@Override
177-
public InputStream getInputStream(String container, String blob) throws ServiceException {
178-
GetBlobResult blobResult = service.getBlob(container, blob);
179-
return blobResult.getContentStream();
154+
public InputStream getInputStream(String container, String blob) throws URISyntaxException, StorageException {
155+
return client.getContainerReference(container).getBlockBlobReference(blob).openInputStream();
180156
}
181157

182158
@Override
@@ -185,21 +161,20 @@ public OutputStream getOutputStream(String container, String blob) throws URISyn
185161
}
186162

187163
@Override
188-
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException, ServiceException {
164+
public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(String container, String keyPath, String prefix) throws URISyntaxException, StorageException {
189165
logger.debug("listBlobsByPrefix container [{}], keyPath [{}], prefix [{}]", container, keyPath, prefix);
190166
ImmutableMap.Builder<String, BlobMetaData> blobsBuilder = ImmutableMap.builder();
191167

192168
CloudBlobContainer blob_container = client.getContainerReference(container);
193169
if (blob_container.exists()) {
194-
Iterable<ListBlobItem> blobs = blob_container.listBlobs(keyPath + prefix);
195-
for (ListBlobItem blob : blobs) {
196-
URI uri = blob.getUri();
170+
for (ListBlobItem blobItem : blob_container.listBlobs(keyPath + prefix)) {
171+
URI uri = blobItem.getUri();
197172
logger.trace("blob url [{}]", uri);
198173
String blobpath = uri.getPath().substring(container.length() + 1);
199-
BlobProperties properties = service.getBlobProperties(container, blobpath).getProperties();
174+
BlobProperties properties = blob_container.getBlockBlobReference(blobpath).getProperties();
200175
String name = blobpath.substring(keyPath.length() + 1);
201-
logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getContentLength());
202-
blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getContentLength()));
176+
logger.trace("blob url [{}], name [{}], size [{}]", uri, name, properties.getLength());
177+
blobsBuilder.put(name, new PlainBlobMetaData(name, properties.getLength()));
203178
}
204179
}
205180

‎src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobContainer.java

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@
1919

2020
package org.elasticsearch.cloud.azure.blobstore;
2121

22-
import com.microsoft.windowsazure.services.core.ServiceException;
23-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.common.Nullable;
2524
import org.elasticsearch.common.blobstore.BlobMetaData;
2625
import org.elasticsearch.common.blobstore.BlobPath;
@@ -70,11 +69,13 @@ public boolean blobExists(String blobName) {
7069
public InputStream openInput(String blobName) throws IOException {
7170
try {
7271
return blobStore.client().getInputStream(blobStore.container(), buildKey(blobName));
73-
} catch (ServiceException e) {
72+
} catch (StorageException e) {
7473
if (e.getHttpStatusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
7574
throw new FileNotFoundException(e.getMessage());
7675
}
7776
throw new IOException(e);
77+
} catch (URISyntaxException e) {
78+
throw new IOException(e);
7879
}
7980
}
8081

@@ -108,7 +109,7 @@ public ImmutableMap<String, BlobMetaData> listBlobsByPrefix(@Nullable String pre
108109

109110
try {
110111
return blobStore.client().listBlobsByPrefix(blobStore.container(), keyPath, prefix);
111-
} catch (URISyntaxException | StorageException | ServiceException e) {
112+
} catch (URISyntaxException | StorageException e) {
112113
logger.warn("can not access [{}] in container {{}}: {}", prefix, blobStore.container(), e.getMessage());
113114
throw new IOException(e);
114115
}

‎src/main/java/org/elasticsearch/cloud/azure/blobstore/AzureBlobStore.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@
1919

2020
package org.elasticsearch.cloud.azure.blobstore;
2121

22-
import com.microsoft.windowsazure.services.core.ServiceException;
23-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.cloud.azure.AzureStorageService;
2524
import org.elasticsearch.common.blobstore.BlobContainer;
2625
import org.elasticsearch.common.blobstore.BlobPath;
@@ -76,7 +75,7 @@ public void delete(BlobPath path) {
7675

7776
try {
7877
client.deleteFiles(container, keyPath);
79-
} catch (URISyntaxException | StorageException | ServiceException e) {
78+
} catch (URISyntaxException | StorageException e) {
8079
logger.warn("can not remove [{}] in container {{}}: {}", keyPath, container, e.getMessage());
8180
}
8281
}

‎src/main/java/org/elasticsearch/repositories/azure/AzureRepository.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
package org.elasticsearch.repositories.azure;
2121

22-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2323
import org.elasticsearch.cloud.azure.AzureStorageService;
2424
import org.elasticsearch.cloud.azure.blobstore.AzureBlobStore;
2525
import org.elasticsearch.common.Strings;

‎src/test/java/org/elasticsearch/repositories/azure/AbstractAzureRepositoryServiceTest.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@
1919

2020
package org.elasticsearch.repositories.azure;
2121

22-
import com.microsoft.windowsazure.services.core.ServiceException;
23-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2423
import org.elasticsearch.cloud.azure.AbstractAzureTest;
2524
import org.elasticsearch.cloud.azure.AzureStorageService;
2625
import org.elasticsearch.cluster.metadata.IndexMetaData;
@@ -87,15 +86,15 @@ public Settings indexSettings() {
8786
}
8887

8988
@Before @After
90-
public final void wipe() throws StorageException, ServiceException, URISyntaxException {
89+
public final void wipe() throws StorageException, URISyntaxException {
9190
wipeRepositories();
9291
cleanRepositoryFiles(basePath);
9392
}
9493

9594
/**
9695
* Purge the test container
9796
*/
98-
public void cleanRepositoryFiles(String path) throws StorageException, ServiceException, URISyntaxException {
97+
public void cleanRepositoryFiles(String path) throws StorageException, URISyntaxException {
9998
String container = internalCluster().getInstance(Settings.class).get("repositories.azure.container");
10099
logger.info("--> remove blobs in container [{}]", container);
101100
AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);

‎src/test/java/org/elasticsearch/repositories/azure/AzureSnapshotRestoreITest.java

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@
2020
package org.elasticsearch.repositories.azure;
2121

2222

23-
import com.microsoft.windowsazure.services.core.ServiceException;
24-
import com.microsoft.windowsazure.services.core.storage.StorageException;
23+
import com.microsoft.azure.storage.StorageException;
2524
import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse;
2625
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
2726
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
@@ -80,7 +79,7 @@ public Settings indexSettings() {
8079
}
8180

8281
@Before
83-
public final void wipeBefore() throws StorageException, ServiceException, URISyntaxException {
82+
public final void wipeBefore() throws StorageException, URISyntaxException {
8483
wipeRepositories();
8584
cleanRepositoryFiles(
8685
getContainerName(),
@@ -89,7 +88,7 @@ public final void wipeBefore() throws StorageException, ServiceException, URISyn
8988
}
9089

9190
@After
92-
public final void wipeAfter() throws StorageException, ServiceException, URISyntaxException {
91+
public final void wipeAfter() throws StorageException, URISyntaxException {
9392
wipeRepositories();
9493
cleanRepositoryFiles(
9594
getContainerName(),
@@ -240,7 +239,7 @@ public void testMultipleRepositories() {
240239
* For issue #26: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/26
241240
*/
242241
@Test
243-
public void testListBlobs_26() throws StorageException, ServiceException, URISyntaxException {
242+
public void testListBlobs_26() throws StorageException, URISyntaxException {
244243
createIndex("test-idx-1", "test-idx-2", "test-idx-3");
245244
ensureGreen();
246245

@@ -299,7 +298,7 @@ public void testListBlobs_26() throws StorageException, ServiceException, URISyn
299298
* For issue #28: https://github.com/elasticsearch/elasticsearch-cloud-azure/issues/28
300299
*/
301300
@Test
302-
public void testGetDeleteNonExistingSnapshot_28() throws StorageException, ServiceException, URISyntaxException {
301+
public void testGetDeleteNonExistingSnapshot_28() throws StorageException, URISyntaxException {
303302
ClusterAdminClient client = client().admin().cluster();
304303
logger.info("--> creating azure repository without any path");
305304
PutRepositoryResponse putRepositoryResponse = client.preparePutRepository("test-repo").setType("azure")
@@ -408,7 +407,7 @@ public static void wipeRepositories(String... repositories) {
408407
/**
409408
* Purge the test containers
410409
*/
411-
public void cleanRepositoryFiles(String... containers) throws StorageException, ServiceException, URISyntaxException {
410+
public void cleanRepositoryFiles(String... containers) throws StorageException, URISyntaxException {
412411
AzureStorageService client = internalCluster().getInstance(AzureStorageService.class);
413412
for (String container : containers) {
414413
logger.info("--> remove container [{}]", container);

‎src/test/java/org/elasticsearch/repositories/azure/AzureStorageServiceMock.java

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
package org.elasticsearch.repositories.azure;
2121

22-
import com.microsoft.windowsazure.services.core.storage.StorageException;
22+
import com.microsoft.azure.storage.StorageException;
2323
import org.elasticsearch.ElasticsearchException;
2424
import org.elasticsearch.cloud.azure.AzureStorageService;
2525
import org.elasticsearch.common.blobstore.BlobMetaData;
@@ -29,7 +29,10 @@
2929
import org.elasticsearch.common.inject.Inject;
3030
import org.elasticsearch.common.settings.Settings;
3131

32-
import java.io.*;
32+
import java.io.ByteArrayInputStream;
33+
import java.io.ByteArrayOutputStream;
34+
import java.io.InputStream;
35+
import java.io.OutputStream;
3336
import java.net.URISyntaxException;
3437
import java.util.Locale;
3538
import java.util.Map;

0 commit comments

Comments
 (0)