Skip to content

Commit a4a924f

Browse files
stewi1014stewi1014
committed
Revert "Merge pull request webbukkit#4030 from ChimneySwift/generic-s3"
This reverts commit b0e56d3, reversing changes made to 20700c2. The reason for reverting this is that it breaks zoom tile generation for Amazon's s3 in an insidious way. No errors are reported, and the zoom tile thread infinitely iterates over calling the s3 API, throwing an error when s3 replies and then swallowing the error and immediately trying again. This generates about 2-3MB/s of useless traffic to s3 and the immense amount of API calls made incurs a significant cost.
1 parent 080a84f commit a4a924f

File tree

4 files changed

+138
-119
lines changed

4 files changed

+138
-119
lines changed

‎DynmapCore/build.gradle

+10-14
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,11 @@ dependencies {
1818
implementation 'org.yaml:snakeyaml:1.23' // DON'T UPDATE - NEWER ONE TRIPS ON WINDOWS ENCODED FILES
1919
implementation 'com.googlecode.owasp-java-html-sanitizer:owasp-java-html-sanitizer:20180219.1'
2020
implementation 'org.postgresql:postgresql:42.2.18'
21-
implementation 'io.github.linktosriram.s3lite:core:0.0.2-SNAPSHOT'
22-
implementation 'io.github.linktosriram.s3lite:api:0.0.2-SNAPSHOT'
23-
implementation 'io.github.linktosriram.s3lite:http-client-url-connection:0.0.2-SNAPSHOT'
24-
implementation 'io.github.linktosriram.s3lite:http-client-spi:0.0.2-SNAPSHOT'
25-
implementation 'io.github.linktosriram.s3lite:util:0.0.2-SNAPSHOT'
26-
implementation 'jakarta.xml.bind:jakarta.xml.bind-api:3.0.1'
27-
implementation 'com.sun.xml.bind:jaxb-impl:3.0.0'
21+
implementation 'io.github.linktosriram:s3-lite-core:0.2.0'
22+
implementation 'io.github.linktosriram:s3-lite-api:0.2.0'
23+
implementation 'io.github.linktosriram:s3-lite-http-client-url-connection:0.2.0'
24+
implementation 'io.github.linktosriram:s3-lite-http-client-spi:0.2.0'
25+
implementation 'io.github.linktosriram:s3-lite-util:0.2.0'
2826
}
2927

3028
processResources {
@@ -59,13 +57,11 @@ shadowJar {
5957
include(dependency('org.eclipse.jetty::'))
6058
include(dependency('org.eclipse.jetty.orbit:javax.servlet:'))
6159
include(dependency('org.postgresql:postgresql:'))
62-
include(dependency('io.github.linktosriram.s3lite:core:'))
63-
include(dependency('io.github.linktosriram.s3lite:api:'))
64-
include(dependency('io.github.linktosriram.s3lite:http-client-url-connection:'))
65-
include(dependency('io.github.linktosriram.s3lite:http-client-spi:'))
66-
include(dependency('io.github.linktosriram.s3lite:util:'))
67-
include(dependency('jakarta.xml.bind::'))
68-
include(dependency('com.sun.xml.bind::'))
60+
include(dependency('io.github.linktosriram:s3-lite-core:'))
61+
include(dependency('io.github.linktosriram:s3-lite-api:'))
62+
include(dependency('io.github.linktosriram:s3-lite-http-client-url-connection:'))
63+
include(dependency('io.github.linktosriram:s3-lite-http-client-spi:'))
64+
include(dependency('io.github.linktosriram:s3-lite-util:'))
6965
include(dependency(':DynmapCoreAPI'))
7066
exclude("META-INF/maven/**")
7167
exclude("META-INF/services/**")

‎DynmapCore/src/main/java/org/dynmap/storage/aws_s3/AWSS3MapStorage.java

+128-103
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,32 @@
11
package org.dynmap.storage.aws_s3;
22

3+
import java.io.IOException;
4+
import java.nio.charset.StandardCharsets;
5+
import java.security.MessageDigest;
6+
import java.security.NoSuchAlgorithmException;
7+
import java.util.ArrayList;
8+
import java.util.Arrays;
9+
import java.util.Collections;
10+
import java.util.List;
11+
import java.util.Map;
12+
import java.util.concurrent.ConcurrentHashMap;
13+
14+
import org.dynmap.DynmapCore;
15+
import org.dynmap.DynmapWorld;
16+
import org.dynmap.Log;
17+
import org.dynmap.MapType;
18+
import org.dynmap.MapType.ImageEncoding;
19+
import org.dynmap.MapType.ImageVariant;
20+
import org.dynmap.PlayerFaces.FaceType;
21+
import org.dynmap.WebAuthManager;
22+
import org.dynmap.storage.MapStorage;
23+
import org.dynmap.storage.MapStorageTile;
24+
import org.dynmap.storage.MapStorageTileEnumCB;
25+
import org.dynmap.storage.MapStorageBaseTileEnumCB;
26+
import org.dynmap.storage.MapStorageTileSearchEndCB;
27+
import org.dynmap.utils.BufferInputStream;
28+
import org.dynmap.utils.BufferOutputStream;
29+
330
import io.github.linktosriram.s3lite.api.client.S3Client;
431
import io.github.linktosriram.s3lite.api.exception.NoSuchKeyException;
532
import io.github.linktosriram.s3lite.api.exception.S3Exception;
@@ -118,19 +145,20 @@ public TileRead read() {
118145

119146
@Override
120147
public boolean write(long hash, BufferOutputStream encImage, long timestamp) {
121-
boolean done = false;
122-
S3Client s3 = null;
123-
try {
124-
s3 = getConnection();
125-
if (encImage == null) { // Delete?
126-
DeleteObjectRequest req = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
127-
s3.deleteObject(req);
128-
} else {
129-
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType())
130-
.addMetadata("x-dynmap-hash", Long.toHexString(hash)).addMetadata("x-dynmap-ts", Long.toString(timestamp)).build();
131-
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
132-
}
133-
done = true;
148+
boolean done = false;
149+
S3Client s3 = null;
150+
try {
151+
s3 = getConnection();
152+
if (encImage == null) { // Delete?
153+
DeleteObjectRequest req = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
154+
s3.deleteObject(req);
155+
}
156+
else {
157+
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(map.getImageFormat().getEncoding().getContentType())
158+
.addMetadata("x-dynmap-hash", Long.toHexString(hash)).addMetadata("x-dynmap-ts", Long.toString(timestamp)).build();
159+
s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
160+
}
161+
done = true;
134162
} catch (S3Exception x) {
135163
Log.severe("AWS Exception", x);
136164
} catch (StorageShutdownException x) {
@@ -214,7 +242,7 @@ public String toString() {
214242
}
215243

216244
private String bucketname;
217-
private Region region;
245+
private String region;
218246
private String access_key_id;
219247
private String secret_access_key;
220248
private String prefix;
@@ -241,22 +269,12 @@ public boolean init(DynmapCore core) {
241269
}
242270
// Get our settings
243271
bucketname = core.configuration.getString("storage/bucketname", "dynmap");
272+
region = core.configuration.getString("storage/region", "us-east-1");
244273
access_key_id = core.configuration.getString("storage/aws_access_key_id", System.getenv("AWS_ACCESS_KEY_ID"));
245274
secret_access_key = core.configuration.getString("storage/aws_secret_access_key", System.getenv("AWS_SECRET_ACCESS_KEY"));
246275
prefix = core.configuration.getString("storage/prefix", "");
247-
248-
// Either use a custom region, or one of the default AWS regions
249-
String region_name = core.configuration.getString("storage/region", "us-east-1");
250-
String region_endpoint = core.configuration.getString("storage/override_endpoint", "");
251-
252-
if (region_endpoint.length() > 0) {
253-
region = Region.of(region_name, URI.create(region_endpoint));
254-
} else {
255-
region = Region.fromString(region_name);
256-
}
257-
258-
if ((prefix.length() > 0) && (prefix.charAt(prefix.length() - 1) != '/')) {
259-
prefix += '/';
276+
if ((prefix.length() > 0) && (prefix.charAt(prefix.length()-1) != '/')) {
277+
prefix += '/';
260278
}
261279
// Now creste the access client for the S3 service
262280
Log.info("Using AWS S3 storage: web site at S3 bucket " + bucketname + " in region " + region);
@@ -503,20 +521,21 @@ public void purgeMapTiles(DynmapWorld world, MapType map) {
503521

504522
@Override
505523
public boolean setPlayerFaceImage(String playername, FaceType facetype,
506-
BufferOutputStream encImage) {
507-
boolean done = false;
508-
String baseKey = prefix + "tiles/faces/" + facetype.id + "/" + playername + ".png";
509-
S3Client s3 = null;
510-
try {
511-
s3 = getConnection();
512-
if (encImage == null) { // Delete?
513-
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
514-
s3.deleteObject(delreq);
515-
} else {
516-
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
517-
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
518-
}
519-
done = true;
524+
BufferOutputStream encImage) {
525+
boolean done = false;
526+
String baseKey = prefix + "tiles/faces/" + facetype.id + "/" + playername + ".png";
527+
S3Client s3 = null;
528+
try {
529+
s3 = getConnection();
530+
if (encImage == null) { // Delete?
531+
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
532+
s3.deleteObject(delreq);
533+
}
534+
else {
535+
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
536+
s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
537+
}
538+
done = true;
520539
} catch (S3Exception x) {
521540
Log.severe("AWS Exception", x);
522541
} catch (StorageShutdownException x) {
@@ -556,19 +575,20 @@ public boolean hasPlayerFaceImage(String playername, FaceType facetype) {
556575

557576
@Override
558577
public boolean setMarkerImage(String markerid, BufferOutputStream encImage) {
559-
boolean done = false;
560-
String baseKey = prefix + "tiles/_markers_/" + markerid + ".png";
561-
S3Client s3 = null;
562-
try {
563-
s3 = getConnection();
564-
if (encImage == null) { // Delete?
565-
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
566-
s3.deleteObject(delreq);
567-
} else {
568-
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
569-
s3.putObject(req, RequestBody.fromBytes(encImage.buf));
570-
}
571-
done = true;
578+
boolean done = false;
579+
String baseKey = prefix + "tiles/_markers_/" + markerid + ".png";
580+
S3Client s3 = null;
581+
try {
582+
s3 = getConnection();
583+
if (encImage == null) { // Delete?
584+
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
585+
s3.deleteObject(delreq);
586+
}
587+
else {
588+
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType("image/png").build();
589+
s3.putObject(req, RequestBody.fromBytes(encImage.buf, encImage.len));
590+
}
591+
done = true;
572592
} catch (S3Exception x) {
573593
Log.severe("AWS Exception", x);
574594
} catch (StorageShutdownException x) {
@@ -688,51 +708,56 @@ public boolean needsStaticWebFiles() {
688708
* @return true if successful
689709
*/
690710
public boolean setStaticWebFile(String fileid, BufferOutputStream content) {
691-
692-
boolean done = false;
693-
String baseKey = prefix + fileid;
694-
S3Client s3 = null;
695-
try {
696-
s3 = getConnection();
697-
byte[] cacheval = standalone_cache.get(fileid);
698-
699-
if (content == null) { // Delete?
700-
if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached?
701-
return true;
702-
}
703-
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
704-
s3.deleteObject(delreq);
705-
standalone_cache.put(fileid, new byte[0]); // Mark in cache
706-
} else {
707-
byte[] digest = content.buf;
708-
try {
709-
MessageDigest md = MessageDigest.getInstance("MD5");
710-
md.update(content.buf);
711-
digest = md.digest();
712-
} catch (NoSuchAlgorithmException nsax) {
713-
714-
}
715-
// If cached and same, just return
716-
if (Arrays.equals(digest, cacheval)) {
717-
return true;
718-
}
719-
String ct = "text/plain";
720-
if (fileid.endsWith(".json")) {
721-
ct = "application/json";
722-
} else if (fileid.endsWith(".php")) {
723-
ct = "application/x-httpd-php";
724-
} else if (fileid.endsWith(".html")) {
725-
ct = "text/html";
726-
} else if (fileid.endsWith(".css")) {
727-
ct = "text/css";
728-
} else if (fileid.endsWith(".js")) {
729-
ct = "application/x-javascript";
730-
}
731-
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build();
732-
s3.putObject(req, RequestBody.fromBytes(content.buf));
733-
standalone_cache.put(fileid, digest);
734-
}
735-
done = true;
711+
712+
boolean done = false;
713+
String baseKey = prefix + fileid;
714+
S3Client s3 = null;
715+
try {
716+
s3 = getConnection();
717+
byte[] cacheval = standalone_cache.get(fileid);
718+
719+
if (content == null) { // Delete?
720+
if ((cacheval != null) && (cacheval.length == 0)) { // Delete cached?
721+
return true;
722+
}
723+
DeleteObjectRequest delreq = DeleteObjectRequest.builder().bucketName(bucketname).key(baseKey).build();
724+
s3.deleteObject(delreq);
725+
standalone_cache.put(fileid, new byte[0]); // Mark in cache
726+
}
727+
else {
728+
byte[] digest = content.buf;
729+
try {
730+
MessageDigest md = MessageDigest.getInstance("MD5");
731+
md.update(content.buf);
732+
digest = md.digest();
733+
} catch (NoSuchAlgorithmException nsax) {
734+
735+
}
736+
// If cached and same, just return
737+
if (Arrays.equals(digest, cacheval)) {
738+
return true;
739+
}
740+
String ct = "text/plain";
741+
if (fileid.endsWith(".json")) {
742+
ct = "application/json";
743+
}
744+
else if (fileid.endsWith(".php")) {
745+
ct = "application/x-httpd-php";
746+
}
747+
else if (fileid.endsWith(".html")) {
748+
ct = "text/html";
749+
}
750+
else if (fileid.endsWith(".css")) {
751+
ct = "text/css";
752+
}
753+
else if (fileid.endsWith(".js")) {
754+
ct = "application/x-javascript";
755+
}
756+
PutObjectRequest req = PutObjectRequest.builder().bucketName(bucketname).key(baseKey).contentType(ct).build();
757+
s3.putObject(req, RequestBody.fromBytes(content.buf, content.len));
758+
standalone_cache.put(fileid, digest);
759+
}
760+
done = true;
736761
} catch (S3Exception x) {
737762
Log.severe("AWS Exception", x);
738763
} catch (StorageShutdownException x) {
@@ -757,10 +782,10 @@ private S3Client getConnection() throws S3Exception, StorageShutdownException {
757782
if (c == null) {
758783
if (cpoolCount < POOLSIZE) { // Still more we can have
759784
c = new DefaultS3ClientBuilder()
760-
.credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key))
761-
.region(region)
762-
.httpClient(URLConnectionSdkHttpClient.create())
763-
.build();
785+
.credentialsProvider(() -> AwsBasicCredentials.create(access_key_id, secret_access_key))
786+
.region(Region.fromString(region))
787+
.httpClient(URLConnectionSdkHttpClient.create())
788+
.build();
764789
if (c == null) {
765790
Log.severe("Error creating S3 access client");
766791
return null;

‎fabric-1.21.1/src/main/resources/configuration.txt

-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ storage:
4747
#aws_access_key_id: "<aws-access-key-id>"
4848
#aws_secret_access_key: "<aws-secret-access-key>"
4949
#prefix: ""
50-
#override_endpoint: ""
5150

5251
components:
5352
- class: org.dynmap.ClientConfigurationComponent

‎forge-1.21/src/main/resources/configuration.txt

-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ storage:
4747
#aws_access_key_id: "<aws-access-key-id>"
4848
#aws_secret_access_key: "<aws-secret-access-key>"
4949
#prefix: ""
50-
#override_endpoint: ""
5150

5251
components:
5352
- class: org.dynmap.ClientConfigurationComponent

0 commit comments

Comments
 (0)