Update the GeoIp database download method [BREAKING] (#1990)

sgdc3 · web-flow · commit a43127dd2b9f · 2020-01-21T10:28:50.000+01:00
* Update the GeoIp database download method [BREAKING] Now GeoIp database updates require a ClientID and a LicenseKey, which can be obtained for free at https://www.maxmind.com/en/accounts/current/license-key * Codestyle
diff --git a/docs/config.md b/docs/config.md
@@ -1,5 +1,5 @@
 <!-- AUTO-GENERATED FILE! Do not edit this directly -->
-<!-- File auto-generated on Thu Oct 17 08:29:25 CEST 2019. See docs/config/config.tpl.md -->
+<!-- File auto-generated on Mon Jan 20 14:16:50 CET 2020. See docs/config/config.tpl.md -->
 
 ## AuthMe Configuration
 The first time you run AuthMe it will create a config.yml file in the plugins/AuthMe folder,
@@ -398,6 +398,12 @@ Protection:
     enableProtection: false
     # Apply the protection also to registered usernames
     enableProtectionRegistered: true
+    geoIpDatabase:
+        # The MaxMind clientId used to download the GeoIp database,
+        # get one at https://www.maxmind.com/en/accounts/current/license-key
+        clientId: ''
+        # The MaxMind licenseKey used to download the GeoIp database.
+        licenseKey: ''
     # Countries allowed to join the server and register. For country codes, see
     # https://dev.maxmind.com/geoip/legacy/codes/iso3166/
     # Use "LOCALHOST" for local addresses.
@@ -576,4 +582,4 @@ To change settings on a running server, save your changes to config.yml and use
 
 ---
 
-This page was automatically generated on the [AuthMe/AuthMeReloaded repository](https://github.com/AuthMe/AuthMeReloaded/tree/master/docs/) on Thu Oct 17 08:29:25 CEST 2019
+This page was automatically generated on the [AuthMe/AuthMeReloaded repository](https://github.com/AuthMe/AuthMeReloaded/tree/master/docs/) on Mon Jan 20 14:16:50 CET 2020
diff --git a/src/main/java/fr/xephi/authme/service/GeoIpService.java b/src/main/java/fr/xephi/authme/service/GeoIpService.java
@@ -4,9 +4,6 @@
 import com.google.common.hash.HashCode;
 import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hashing;
-import com.google.common.io.Resources;
-import com.ice.tar.TarEntry;
-import com.ice.tar.TarInputStream;
 import com.maxmind.db.GeoIp2Provider;
 import com.maxmind.db.Reader;
 import com.maxmind.db.Reader.FileMode;
@@ -16,19 +13,19 @@
 import fr.xephi.authme.ConsoleLogger;
 import fr.xephi.authme.initialization.DataFolder;
 import fr.xephi.authme.output.ConsoleLoggerFactory;
+import fr.xephi.authme.settings.Settings;
+import fr.xephi.authme.settings.properties.ProtectionSettings;
 import fr.xephi.authme.util.FileUtils;
 import fr.xephi.authme.util.InternetProtocolUtils;
 
 import javax.inject.Inject;
 import java.io.BufferedInputStream;
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.net.HttpURLConnection;
 import java.net.InetAddress;
 import java.net.URL;
 import java.net.UnknownHostException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
@@ -38,6 +35,7 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
+import java.util.Base64;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.zip.GZIPInputStream;
@@ -48,39 +46,38 @@ public class GeoIpService {
             "[LICENSE] This product includes GeoLite2 data created by MaxMind, available at https://www.maxmind.com";
 
     private static final String DATABASE_NAME = "GeoLite2-Country";
-    private static final String DATABASE_EXT = ".mmdb";
-    private static final String DATABASE_FILE = DATABASE_NAME + DATABASE_EXT;
+    private static final String DATABASE_FILE = DATABASE_NAME + ".mmdb";
+    private static final String DATABASE_TMP_FILE = DATABASE_NAME + ".mmdb.tmp";
 
-    private static final String ARCHIVE_FILE = DATABASE_NAME + ".tar.gz";
+    private static final String ARCHIVE_FILE = DATABASE_NAME + ".mmdb.gz";
 
-    private static final String ARCHIVE_URL = "https://geolite.maxmind.com/download/geoip/database/" + ARCHIVE_FILE;
-    private static final String CHECKSUM_URL = ARCHIVE_URL + ".md5";
+    private static final String ARCHIVE_URL =
+        "https://updates.maxmind.com/geoip/databases/" + DATABASE_NAME + "/update";
 
     private static final int UPDATE_INTERVAL_DAYS = 30;
 
-    // The server for MaxMind doesn't seem to understand RFC1123,
-    // but every HTTP implementation have to support  RFC 1023
-    private static final String TIME_RFC_1023 = "EEE, dd-MMM-yy HH:mm:ss zzz";
-
     private final ConsoleLogger logger = ConsoleLoggerFactory.get(GeoIpService.class);
     private final Path dataFile;
     private final BukkitService bukkitService;
+    private final Settings settings;
 
     private GeoIp2Provider databaseReader;
     private volatile boolean downloading;
 
     @Inject
-    GeoIpService(@DataFolder File dataFolder, BukkitService bukkitService) {
+    GeoIpService(@DataFolder File dataFolder, BukkitService bukkitService, Settings settings) {
         this.bukkitService = bukkitService;
         this.dataFile = dataFolder.toPath().resolve(DATABASE_FILE);
+        this.settings = settings;
 
         // Fires download of recent data or the initialization of the look up service
         isDataAvailable();
     }
 
     @VisibleForTesting
-    GeoIpService(@DataFolder File dataFolder, BukkitService bukkitService, GeoIp2Provider reader) {
+    GeoIpService(@DataFolder File dataFolder, BukkitService bukkitService, Settings settings, GeoIp2Provider reader) {
         this.bukkitService = bukkitService;
+        this.settings = settings;
         this.dataFile = dataFolder.toPath().resolve(DATABASE_FILE);
 
         this.databaseReader = reader;
@@ -135,22 +132,26 @@ private void updateDatabase() {
         logger.info("Downloading GEO IP database, because the old database is older than "
                 + UPDATE_INTERVAL_DAYS + " days or doesn't exist");
 
+        Path downloadFile = null;
         Path tempFile = null;
         try {
             // download database to temporarily location
-            tempFile = Files.createTempFile(ARCHIVE_FILE, null);
-            if (!downloadDatabaseArchive(tempFile)) {
+            downloadFile = Files.createTempFile(ARCHIVE_FILE, null);
+            tempFile = Files.createTempFile(DATABASE_TMP_FILE, null);
+            String expectedChecksum = downloadDatabaseArchive(downloadFile);
+            if (expectedChecksum == null) {
                 logger.info("There is no newer GEO IP database uploaded to MaxMind. Using the old one for now.");
                 startReading();
                 return;
             }
 
+            // tar extract database and copy to target destination
+            extractDatabase(downloadFile, tempFile);
+
             // MD5 checksum verification
-            String expectedChecksum = Resources.toString(new URL(CHECKSUM_URL), StandardCharsets.UTF_8);
             verifyChecksum(Hashing.md5(), tempFile, expectedChecksum);
 
-            // tar extract database and copy to target destination
-            extractDatabase(tempFile, dataFile);
+            Files.copy(tempFile, dataFile);
 
             //only set this value to false on success otherwise errors could lead to endless download triggers
             logger.info("Successfully downloaded new GEO IP database to " + dataFile);
@@ -159,6 +160,9 @@ private void updateDatabase() {
             logger.logException("Could not download GeoLiteAPI database", ioEx);
         } finally {
             // clean up
+            if (downloadFile != null) {
+                FileUtils.delete(downloadFile.toFile());
+            }
             if (tempFile != null) {
                 FileUtils.delete(tempFile.toFile());
             }
@@ -178,36 +182,51 @@ private void startReading() throws IOException {
      *
      * @param lastModified modification timestamp of the already present file
      * @param destination save file
-     * @return false if we already have the newest version, true if successful
+     * @return null if no updates were found, the MD5 hash of the downloaded archive if successful
      * @throws IOException if failed during downloading and writing to destination file
      */
-    private boolean downloadDatabaseArchive(Instant lastModified, Path destination) throws IOException {
+    private String downloadDatabaseArchive(Instant lastModified, Path destination) throws IOException {
         HttpURLConnection connection = (HttpURLConnection) new URL(ARCHIVE_URL).openConnection();
+
+        String clientId = settings.getProperty(ProtectionSettings.MAXMIND_API_CLIENT_ID);
+        String licenseKey = settings.getProperty(ProtectionSettings.MAXMIND_API_LICENSE_KEY);
+        if (clientId.isEmpty() || licenseKey.isEmpty()) {
+            logger.warning("No MaxMind credentials found in the configuration file!"
+                + " GeoIp protections will be disabled.");
+            return null;
+        }
+        String basicAuth = "Basic " + new String(Base64.getEncoder().encode((clientId + ":" + licenseKey).getBytes()));
+        connection.setRequestProperty("Authorization", basicAuth);
+
         if (lastModified != null) {
             // Only download if we actually need a newer version - this field is specified in GMT zone
             ZonedDateTime zonedTime = lastModified.atZone(ZoneId.of("GMT"));
-            String timeFormat = DateTimeFormatter.ofPattern(TIME_RFC_1023).format(zonedTime);
+            String timeFormat = DateTimeFormatter.RFC_1123_DATE_TIME.format(zonedTime);
             connection.addRequestProperty("If-Modified-Since", timeFormat);
         }
 
         if (connection.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
             //we already have the newest version
             connection.getInputStream().close();
-            return false;
+            return null;
         }
 
+        String hash = connection.getHeaderField("X-Database-MD5");
+        String rawModifiedDate = connection.getHeaderField("Last-Modified");
+        Instant modifiedDate = Instant.from(DateTimeFormatter.RFC_1123_DATE_TIME.parse(rawModifiedDate));
         Files.copy(connection.getInputStream(), destination, StandardCopyOption.REPLACE_EXISTING);
-        return true;
+        Files.setLastModifiedTime(destination, FileTime.from(modifiedDate));
+        return hash;
     }
 
     /**
      * Downloads the archive to the destination file if it's newer than the locally version.
      *
      * @param destination save file
-     * @return false if we already have the newest version, true if successful
+     * @return null if no updates were found, the MD5 hash of the downloaded archive if successful
      * @throws IOException if failed during downloading and writing to destination file
      */
-    private boolean downloadDatabaseArchive(Path destination) throws IOException {
+    private String downloadDatabaseArchive(Path destination) throws IOException {
         Instant lastModified = null;
         if (Files.exists(dataFile)) {
             lastModified = Files.getLastModifiedTime(dataFile).toInstant();
@@ -234,34 +253,23 @@ private void verifyChecksum(HashFunction function, Path file, String expectedChe
     }
 
     /**
-     * Extract the database from the tar archive. Existing outputFile will be replaced if it already exists.
+     * Extract the database from gzipped data. Existing outputFile will be replaced if it already exists.
      *
-     * @param tarInputFile gzipped tar input file where the database is
+     * @param inputFile gzipped database input file
      * @param outputFile destination file for the database
-     * @throws IOException on I/O error reading the tar archive, or writing the output
-     * @throws FileNotFoundException if the database cannot be found inside the archive
+     * @throws IOException on I/O error reading the archive, or writing the output
      */
-    private void extractDatabase(Path tarInputFile, Path outputFile) throws FileNotFoundException, IOException {
+    private void extractDatabase(Path inputFile, Path outputFile) throws IOException {
         // .gz -> gzipped file
-        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(tarInputFile));
-             TarInputStream tarIn = new TarInputStream(new GZIPInputStream(in))) {
-            for (TarEntry entry = tarIn.getNextEntry(); entry != null; entry = tarIn.getNextEntry()) {
-                // filename including folders (absolute path inside the archive)
-                String filename = entry.getName();
-                if (entry.isDirectory() || !filename.endsWith(DATABASE_EXT)) {
-                    continue;
-                }
+        try (BufferedInputStream in = new BufferedInputStream(Files.newInputStream(inputFile));
+             GZIPInputStream gzipIn = new GZIPInputStream(in)) {
 
-                // found the database file and copy file
-                Files.copy(tarIn, outputFile, StandardCopyOption.REPLACE_EXISTING);
+            // found the database file and copy file
+            Files.copy(gzipIn, outputFile, StandardCopyOption.REPLACE_EXISTING);
 
-                // update the last modification date to be same as in the archive
-                Files.setLastModifiedTime(outputFile, FileTime.from(entry.getModTime().toInstant()));
-                return;
-            }
+            // update the last modification date to be same as in the archive
+            Files.setLastModifiedTime(outputFile, Files.getLastModifiedTime(inputFile));
         }
-
-        throw new FileNotFoundException("Cannot find database inside downloaded GEO IP file at " + tarInputFile);
     }
 
     /**
diff --git a/src/main/java/fr/xephi/authme/settings/properties/ProtectionSettings.java b/src/main/java/fr/xephi/authme/settings/properties/ProtectionSettings.java
@@ -20,6 +20,15 @@ public final class ProtectionSettings implements SettingsHolder {
     public static final Property<Boolean> ENABLE_PROTECTION_REGISTERED =
         newProperty("Protection.enableProtectionRegistered", true);
 
+    @Comment({"The MaxMind clientId used to download the GeoIp database,",
+        "get one at https://www.maxmind.com/en/accounts/current/license-key"})
+    public static final Property<String> MAXMIND_API_CLIENT_ID =
+        newProperty("Protection.geoIpDatabase.clientId", "");
+
+    @Comment("The MaxMind licenseKey used to download the GeoIp database.")
+    public static final Property<String> MAXMIND_API_LICENSE_KEY =
+        newProperty("Protection.geoIpDatabase.licenseKey", "");
+
     @Comment({
         "Countries allowed to join the server and register. For country codes, see",
         "https://dev.maxmind.com/geoip/legacy/codes/iso3166/",
diff --git a/src/test/java/fr/xephi/authme/service/GeoIpServiceTest.java b/src/test/java/fr/xephi/authme/service/GeoIpServiceTest.java
@@ -8,6 +8,7 @@
 import java.io.IOException;
 import java.net.InetAddress;
 
+import fr.xephi.authme.settings.Settings;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -40,13 +41,16 @@ public class GeoIpServiceTest {
     @Mock
     private BukkitService bukkitService;
 
+    @Mock
+    private Settings settings;
+
     @Rule
     public TemporaryFolder temporaryFolder = new TemporaryFolder();
 
     @Before
     public void initializeGeoLiteApi() throws IOException {
         dataFolder = temporaryFolder.newFolder();
-        geoIpService = new GeoIpService(dataFolder, bukkitService, lookupService);
+        geoIpService = new GeoIpService(dataFolder, bukkitService, settings, lookupService);
     }
 
     @Test
