Drasticly improve zoom tile calculation for larger maps when using MySQL/MariaDB storage Engine

DynmapCore/src/main/java/org/dynmap/storage/mysql/MySQLMapStorage.java

@@ -809,29 +809,21 @@ private void processEnumMapTiles(DynmapWorld world, MapType map, ImageVariant va
         }
         try {
             c = getConnection();
-            boolean done = false;
-            int limit = 100;
-            int offset = 0;
-            while (!done) {
-	            // Query tiles for given mapkey
-	            Statement stmt = c.createStatement();
-	            ResultSet rs = stmt.executeQuery(String.format("SELECT x,y,zoom,Format FROM %s WHERE MapID=%d LIMIT %d OFFSET %d;", tableTiles, mapkey, limit, offset));
-	            int cnt = 0;
-	            while (rs.next()) {
-	                StorageTile st = new StorageTile(world, map, rs.getInt("x"), rs.getInt("y"), rs.getInt("zoom"), var);
-	                final MapType.ImageEncoding encoding = MapType.ImageEncoding.fromOrd(rs.getInt("Format"));
-	                if(cb != null)
-	                    cb.tileFound(st, encoding);
-	                if(cbBase != null && st.zoom == 0)
-	                    cbBase.tileFound(st, encoding);
-	                st.cleanup();
-	                cnt++;
-	            }
-	            rs.close();
-	            stmt.close();
-	            if (cnt < limit) done = true;
-	            offset += cnt;
+            Statement stmt = c.createStatement(java.sql.ResultSet.TYPE_FORWARD_ONLY, //we want to stream our resultset one row at a time, we are not interessted in going back
+                    java.sql.ResultSet.CONCUR_READ_ONLY); //since we do not handle the entire resultset in memory -> tell the statement that we are going to work read only
+            stmt.setFetchSize(100); //we can change the jdbc "retrieval chunk size". Basicly we limit how much rows are kept in memory. Bigger value = less network calls to DB, but more memory consumption
+            ResultSet rs = stmt.executeQuery(String.format("SELECT x,y,zoom,Format FROM %s WHERE MapID=%d;", tableTiles, mapkey)); //we do the query, but do not set any limit / offset. Since data is not kept in memory, just streamed from DB this should not be a problem, only the rows from setFetchSize are kept in memory.
+            while (rs.next()) {
+                StorageTile st = new StorageTile(world, map, rs.getInt("x"), rs.getInt("y"), rs.getInt("zoom"), var);
+                final MapType.ImageEncoding encoding = MapType.ImageEncoding.fromOrd(rs.getInt("Format"));
+                if(cb != null)
+                    cb.tileFound(st, encoding);
+                if(cbBase != null && st.zoom == 0)
+                    cbBase.tileFound(st, encoding);
+                st.cleanup();
             }
+            rs.close();
+            stmt.close();
             if(cbEnd != null)
                 cbEnd.searchEnded();
         } catch (SQLException x) {

DynmapCore/src/main/java/org/dynmap/utils/ImageIOManager.java

@@ -107,68 +107,126 @@ private static BufferedImage doWEBPDecode(BufferInputStream buf) throws IOExcept
     }
 
     public static BufferOutputStream imageIOEncode(BufferedImage img, ImageFormat fmt) {
+        if(isRequiredJDKVersion(17,-1,-1)){
+            return imageIOEncodeUnsafe(img, fmt); //we can skip Thread safety for more performance
+        }
+        synchronized(imageioLock) {
+            return imageIOEncodeUnsafe(img, fmt);
+        }
+    }
+    private static BufferOutputStream imageIOEncodeUnsafe(BufferedImage img, ImageFormat fmt) {
         BufferOutputStream bos = new BufferOutputStream();
+        try {
+            ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */
 
-        synchronized(imageioLock) {
-            try {
-                ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */
-
-                fmt = validateFormat(fmt);
-
-                if(fmt.getEncoding() == ImageEncoding.JPG) {
-                    WritableRaster raster = img.getRaster();
-                    WritableRaster newRaster = raster.createWritableChild(0, 0, img.getWidth(),
-                            img.getHeight(), 0, 0, new int[] {0, 1, 2});
-                    DirectColorModel cm = (DirectColorModel)img.getColorModel();
-                    DirectColorModel newCM = new DirectColorModel(cm.getPixelSize(),
-                            cm.getRedMask(), cm.getGreenMask(), cm.getBlueMask());
-                    // now create the new buffer that is used ot write the image:
-                    BufferedImage rgbBuffer = new BufferedImage(newCM, newRaster, false, null);
-
-                    // Find a jpeg writer
-                    ImageWriter writer = null;
-                    Iterator<ImageWriter> iter = ImageIO.getImageWritersByFormatName("jpg");
-                    if (iter.hasNext()) {
-                        writer = iter.next();
-                    }
-                    if(writer == null) {
-                        Log.severe("No JPEG ENCODER - Java VM does not support JPEG encoding");
-                        return null;
-                    }
-                    ImageWriteParam iwp = writer.getDefaultWriteParam();
-                    iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
-                    iwp.setCompressionQuality(fmt.getQuality());
-
-                    ImageOutputStream ios;
-                    ios = ImageIO.createImageOutputStream(bos);
-                    writer.setOutput(ios);
-
-                    writer.write(null, new IIOImage(rgbBuffer, null, null), iwp);
-                    writer.dispose();
-
-                    rgbBuffer.flush();
-                }
-                else if (fmt.getEncoding() == ImageEncoding.WEBP) {
-                	doWEBPEncode(img, fmt, bos);                	
+            fmt = validateFormat(fmt);
+
+            if(fmt.getEncoding() == ImageEncoding.JPG) {
+                WritableRaster raster = img.getRaster();
+                WritableRaster newRaster = raster.createWritableChild(0, 0, img.getWidth(),
+                        img.getHeight(), 0, 0, new int[] {0, 1, 2});
+                DirectColorModel cm = (DirectColorModel)img.getColorModel();
+                DirectColorModel newCM = new DirectColorModel(cm.getPixelSize(),
+                        cm.getRedMask(), cm.getGreenMask(), cm.getBlueMask());
+                // now create the new buffer that is used ot write the image:
+                BufferedImage rgbBuffer = new BufferedImage(newCM, newRaster, false, null);
+
+                // Find a jpeg writer
+                ImageWriter writer = null;
+                Iterator<ImageWriter> iter = ImageIO.getImageWritersByFormatName("jpg");
+                if (iter.hasNext()) {
+                    writer = iter.next();
                 }
-                else {
-                    ImageIO.write(img, fmt.getFileExt(), bos); /* Write to byte array stream - prevent bogus I/O errors */
+                if(writer == null) {
+                    Log.severe("No JPEG ENCODER - Java VM does not support JPEG encoding");
+                    return null;
                 }
-            } catch (IOException iox) {
-                Log.info("Error encoding image - " + iox.getMessage());
-                return null;
+                ImageWriteParam iwp = writer.getDefaultWriteParam();
+                iwp.setCompressionMode(ImageWriteParam.MODE_EXPLICIT);
+                iwp.setCompressionQuality(fmt.getQuality());
+
+                ImageOutputStream ios;
+                ios = ImageIO.createImageOutputStream(bos);
+                writer.setOutput(ios);
+
+                writer.write(null, new IIOImage(rgbBuffer, null, null), iwp);
+                writer.dispose();
+
+                rgbBuffer.flush();
+            }
+            else if (fmt.getEncoding() == ImageEncoding.WEBP) {
+            	doWEBPEncode(img, fmt, bos);
             }
+            else {
+                ImageIO.write(img, fmt.getFileExt(), bos); /* Write to byte array stream - prevent bogus I/O errors */
+            }
+        } catch (IOException iox) {
+            Log.info("Error encoding image - " + iox.getMessage());
+            return null;
         }
         return bos;
     }
-    
+
     public static BufferedImage imageIODecode(MapStorageTile.TileRead tr) throws IOException {
+        if(isRequiredJDKVersion(17,-1,-1)){
+            return imageIODecodeUnsafe(tr); //we can skip Thread safety for more performance
+        }
         synchronized(imageioLock) {
-            ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */
-            if (tr.format == ImageEncoding.WEBP) {
-                return doWEBPDecode(tr.image);
+            return imageIODecodeUnsafe(tr);
+        }
+    }
+
+    private static BufferedImage imageIODecodeUnsafe(MapStorageTile.TileRead tr) throws IOException {
+        ImageIO.setUseCache(false); /* Don't use file cache - too small to be worth it */
+        if (tr.format == ImageEncoding.WEBP) {
+            return doWEBPDecode(tr.image);
+        }
+        return ImageIO.read(tr.image);
+    }
+
+    /**
+     * Checks if the current JDK is running at least a specific version
+     * targetMinor and targetBuild can be set to -1, if the java.version only provides a Major release this will then only check for the major release
+     * @param targetMajor the required minimum major version
+     * @param targetMinor the required minimum minor version
+     * @param targetBuild the required minimum build version
+     * @return true if the current JDK version is the required minimum version
+     */
+    private static boolean isRequiredJDKVersion(int targetMajor, int targetMinor, int targetBuild){
+        String javaVersion = System.getProperty("java.version");
+        String[] versionParts = javaVersion.split("\\.");
+        if(versionParts.length < 3){
+            if(versionParts.length == 1
+                    && targetMinor == -1
+                    && targetBuild == -1
+                    && parseInt(versionParts[0], -1) >= targetMajor){
+                return true;//we only have a major version and thats ok
             }
-            return ImageIO.read(tr.image);
+            return false; //can not evaluate
         }
+        int major = parseInt(versionParts[0], -1);
+        int minor = parseInt(versionParts[1], -1);
+        int build = parseInt(versionParts[2], -1);
+        if(major != -1 && major >= targetMajor &&
+                minor != -1 && minor >= targetMinor &&
+                build != -1 && build >= targetBuild
+        ){
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Parses a string to int, with a dynamic fallback value if not parsable
+     * @param input the String to parse
+     * @param fallback the Fallback value to use
+     * @return the parsed integer or the fallback value if unparsable
+     */
+    private static int parseInt(String input, int fallback){
+        int output = fallback;
+        try{
+            output = Integer.parseInt(input);
+        } catch (NumberFormatException e) {}
+        return output;
     }
 }

DynmapCoreAPI/build.gradle

@@ -6,6 +6,7 @@ eclipse {
         name = "Dynmap(DynmapCoreAPI)"
     }
 }
+sourceCompatibility = targetCompatibility = compileJava.sourceCompatibility = compileJava.targetCompatibility = '1.8' // Need this here so eclipse task generates correctly.
 
 description = "DynmapCoreAPI"