Add support for MaxMind GeoIP2 Enterprise and Anonymous-IP databases (#223)

This commit added support for MaxMind GeoIP2 Enterprise and Anonymous-IP databases. 
In addition to that, It also:
- Changed the database type detection to use the same strategy so it doesn't rely on specific database types.
- Updated MaxMind dependencies.
- Added unit tests for the Java classes, including the verification of the plugin's basic operations against all supported databases.
- Changed to get the gem version from the `VERSION` file.

Author: edmocosta

CHANGELOG.md

@@ -1,3 +1,8 @@
+## 7.3.0
+  - Added support for MaxMind GeoIP2 Enterprise and Anonymous-IP databases ([#223](https://github.com/logstash-plugins/logstash-filter-geoip/pull/223))
+  - Updated MaxMind dependencies.
+  - Added tests for the Java classes.
+
 ## 7.2.13
   - [DOC] Add documentation for database auto-update configuration [#210](https://github.com/logstash-plugins/logstash-filter-geoip/pull/210)
 

VERSION

@@ -0,0 +1 @@
+7.3.0

build.gradle

@@ -22,11 +22,17 @@ apply plugin: "java"
 apply plugin: "distribution"
 apply plugin: "idea"
 
-// TODO(sissel): Move this to a file shared by the gemspec.
 group "org.logstash.filters"
-version "6.0.0"
+version "${new File("VERSION").text.trim()}"
 project.archivesBaseName = "logstash-filter-geoip"
 
+String junitVersion = '5.9.2'
+String maxmindGeoip2Version = '2.17.0'
+String maxmindDbVersion = '2.1.0'
+String log4jVersion = '2.17.1'
+String jrubyCompleteVersion = '9.1.13.0'
+String mockitoVersion = '4.11.0'
+
 sourceCompatibility = JavaVersion.VERSION_1_8
 targetCompatibility = JavaVersion.VERSION_1_8
 
@@ -53,29 +59,38 @@ configurations {
 }
 
 dependencies {
-  compileOnly group: "org.apache.logging.log4j", name: "log4j-api", version: "2.17.1"
-  compileOnly group: "org.apache.logging.log4j", name: "log4j-core", version: "2.17.1"
-  compileOnly group: "com.maxmind.geoip2", name: "geoip2", version: "2.9.0"
-  compileOnly group: "com.maxmind.db", name: "maxmind-db", version: "1.2.2"
-  compileOnly group: 'org.jruby', name: 'jruby-complete', version: "1.7.26"
+  compileOnly group: "org.apache.logging.log4j", name: "log4j-api", version: log4jVersion
+  compileOnly group: "org.apache.logging.log4j", name: "log4j-core", version: log4jVersion
+  compileOnly group: 'org.jruby', name: 'jruby-complete', version: jrubyCompleteVersion
   compileOnly fileTree(dir: logstashCoreGemPath, include: '**/*.jar')
 
-  runtimeOnly group: "com.maxmind.geoip2", name: "geoip2", version: "2.9.0"
-  runtimeOnly group: "com.maxmind.db", name: "maxmind-db", version: "1.2.2"
+  implementation group: "com.maxmind.geoip2", name: "geoip2", version: maxmindGeoip2Version
+  implementation group: "com.maxmind.db", name: "maxmind-db", version: maxmindDbVersion
 
-  testImplementation group: 'junit', name: 'junit', version: '4.12'
-  testImplementation group: "org.apache.logging.log4j", name: "log4j-api", version: "2.17.1"
-  testImplementation group: "org.apache.logging.log4j", name: "log4j-core", version: "2.17.1"
-  testImplementation group: 'org.jruby', name: 'jruby-complete', version: "1.7.26"
-  testImplementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.9.0"
-  testImplementation group: "com.maxmind.db", name: "maxmind-db", version: "1.2.2"
-  testImplementation fileTree(dir: logstashCoreGemPath, include: '**/*.jar')
+  testImplementation group: "org.junit.jupiter", name: "junit-jupiter-api", version: junitVersion
+  testImplementation group: "org.junit.jupiter", name: "junit-jupiter-params", version: junitVersion
+  testRuntimeOnly group: "org.junit.jupiter", name: "junit-jupiter-engine", version: junitVersion
+  testImplementation group: "org.apache.logging.log4j", name: "log4j-api", version: log4jVersion
+  testImplementation group: 'org.jruby', name: 'jruby-complete', version: jrubyCompleteVersion
+  testImplementation group: "org.mockito", name: "mockito-core", version: mockitoVersion
 
   geolite2('org.elasticsearch:geolite2-databases:20191119') {
     transitive = false
   }
 }
 
+test {
+  useJUnitPlatform()
+
+  testLogging {
+    events "started", "passed", "skipped", "failed", "standard_out", "standard_error"
+  }
+}
+
+configurations {
+  testImplementation.extendsFrom compileOnly
+}
+
 task generateGemJarRequiresFile {
   doLast {
     File jars_file = file("lib/${project.archivesBaseName}_jars.rb")

docs/index.asciidoc

@@ -190,37 +190,44 @@ When ECS compatibility is enabled, the fields are structured to fit into an ECS
 |===========================
 | Database Field Name | ECS Field | Example
 
-| `ip`               | `[ip]`                     | `12.34.56.78`
-
-| `city_name`        | `[geo][city_name]`         | `Seattle`
-| `country_name`     | `[geo][country_name]`      | `United States`
-| `continent_code`   | `[geo][continent_code]`    | `NA`
-| `continent_name`   | `[geo][continent_name]`    | `North America`
-| `country_code2`    | `[geo][country_iso_code]`  | `US`
-| `country_code3`    | _N/A_                      | `US`
-
-                                                    _maintained for legacy
-                                                    support, but populated
-                                                    with 2-character country
-                                                    code_
-
-| `postal_code`      | `[geo][postal_code]`       | `98106`
-| `region_name`      | `[geo][region_name]`       | `Washington`
-| `region_code`      | `[geo][region_code]`       | `WA`
-| `region_iso_code`* | `[geo][region_iso_code]`   | `US-WA`
-| `timezone`         | `[geo][timezone]`          | `America/Los_Angeles`
-| `location`*        | `[geo][location]`          | `{"lat": 47.6062, "lon": -122.3321}"`
-| `latitude`         | `[geo][location][lat]`     | `47.6062`
-| `longitude`        | `[geo][location][lon]`     | `-122.3321`
-
-| `domain`           | `[domain]`                 | `example.com`
-
-| `asn`              | `[as][number]`             | `98765`
-| `as_org`           | `[as][organization][name]` | `Elastic, NV`
-
-| `isp`              | `[mmdb][isp]`              | `InterLink Supra LLC`
-| `dma_code`         | `[mmdb][dma_code]`         | `819`
-| `organization`     | `[mmdb][organization]`     | `Elastic, NV`
+| `ip`                | `[ip]`                           | `12.34.56.78`
+| `anonymous`         | `[ip_traits][anonymous]`         | `false`
+| `anonymous_vpn`     | `[ip_traits][anonymous_vpn]`     | `false`
+| `hosting_provider`  | `[ip_traits][hosting_provider]`  | `true`
+| `network`           | `[ip_traits][network]`           | `12.34.56.78/20`
+| `public_proxy`      | `[ip_traits][public_proxy]`      | `true`
+| `residential_proxy` | `[ip_traits][residential_proxy]` | `false`
+| `tor_exit_node`     | `[ip_traits][tor_exit_node]`     | `true`
+
+| `city_name`         | `[geo][city_name]`               | `Seattle`
+| `country_name`      | `[geo][country_name]`            | `United States`
+| `continent_code`    | `[geo][continent_code]`          | `NA`
+| `continent_name`    | `[geo][continent_name]`          | `North America`
+| `country_code2`     | `[geo][country_iso_code]`        | `US`
+| `country_code3`     | _N/A_                            | `US`
+
+                                                           _maintained for legacy
+                                                           support, but populated
+                                                           with 2-character country
+                                                           code_
+
+| `postal_code`       | `[geo][postal_code]`             | `98106`
+| `region_name`       | `[geo][region_name]`             | `Washington`
+| `region_code`       | `[geo][region_code]`             | `WA`
+| `region_iso_code`*  | `[geo][region_iso_code]`         | `US-WA`
+| `timezone`          | `[geo][timezone]`                | `America/Los_Angeles`
+| `location`*         | `[geo][location]`                | `{"lat": 47.6062, "lon": -122.3321}"`
+| `latitude`          | `[geo][location][lat]`           | `47.6062`
+| `longitude`         | `[geo][location][lon]`           | `-122.3321`
+
+| `domain`            | `[domain]`                       | `example.com`
+
+| `asn`               | `[as][number]`                   | `98765`
+| `as_org`            | `[as][organization][name]`       | `Elastic, NV`
+
+| `isp`               | `[mmdb][isp]`                    | `InterLink Supra LLC`
+| `dma_code`          | `[mmdb][dma_code]`               | `819`
+| `organization`      | `[mmdb][organization]`           | `Elastic, NV`
 |===========================
 
 NOTE: `*` indicates a composite field, which is only populated if GeoIP lookup result contains all components.
@@ -301,7 +308,7 @@ number of cache misses and waste memory.
 The path to MaxMind's database file that Logstash should use.
 The default database is `GeoLite2-City`.
 This plugin supports several free databases (`GeoLite2-City`, `GeoLite2-Country`, `GeoLite2-ASN`)
-and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`).
+and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`, `GeoIP2-Domain`, `GeoIP2-Enterprise`, `GeoIP2-Anonymous-IP`).
 
 Database auto-update applies to the default distribution.
 When `database` points to user's database path, auto-update is disabled.

lib/logstash-filter-geoip_jars.rb

@@ -1,6 +1,6 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('com.maxmind.geoip2', 'geoip2', '2.9.0')
-require_jar('com.maxmind.db', 'maxmind-db', '1.2.2')
-require_jar('org.logstash.filters', 'logstash-filter-geoip', '6.0.0')
+require_jar('com.maxmind.geoip2', 'geoip2', '2.17.0')
+require_jar('com.maxmind.db', 'maxmind-db', '2.1.0')
+require_jar('org.logstash.filters', 'logstash-filter-geoip', '7.3.0')

lib/logstash/filters/geoip.rb

@@ -171,7 +171,17 @@ def fail_filter
   end
 
   def close
-    @database_manager.unsubscribe_database_path(@default_database_type, self) if @database_manager
+    begin
+      @database_manager.unsubscribe_database_path(@default_database_type, self) if @database_manager
+    rescue => e
+      @logger.error("Error unsubscribing geoip database path", :path => @database, :exception => e)
+    end
+
+    begin
+      @geoipfilter.close if @geoipfilter
+    rescue => e
+      @logger.error("Error closing GeoIPFilter",  :exception => e)
+    end
   end
 
   def select_database_path

logstash-filter-geoip.gemspec

@@ -1,7 +1,9 @@
+VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(VERSION)
+
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '7.2.13'
+  s.version         = VERSION
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Adds geographical information about an IP address"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

src/main/java/org/logstash/filters/geoip/Database.java

@@ -0,0 +1,125 @@
+package org.logstash.filters.geoip;
+
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.Set;
+
+enum Database {
+
+    CITY(
+            "City",
+            EnumSet.of(
+                    Field.IP,
+                    Field.CITY_NAME,
+                    Field.CONTINENT_CODE,
+                    Field.COUNTRY_NAME,
+                    Field.COUNTRY_CODE2,
+                    Field.COUNTRY_CODE3,
+                    Field.POSTAL_CODE,
+                    Field.DMA_CODE,
+                    Field.REGION_NAME,
+                    Field.REGION_ISO_CODE,
+                    Field.TIMEZONE,
+                    Field.LOCATION,
+                    Field.LATITUDE,
+                    Field.LONGITUDE
+            )
+    ),
+    COUNTRY(
+            "Country",
+            EnumSet.of(
+                    Field.IP,
+                    Field.COUNTRY_CODE2,
+                    Field.COUNTRY_NAME,
+                    Field.CONTINENT_NAME
+            )
+    ),
+    DOMAIN(
+            "GeoIP2-Domain",
+            EnumSet.of(
+                    Field.DOMAIN
+            )
+    ),
+    ASN(
+            "GeoLite2-ASN",
+            EnumSet.of(
+                    Field.IP,
+                    Field.AUTONOMOUS_SYSTEM_NUMBER,
+                    Field.AUTONOMOUS_SYSTEM_ORGANIZATION
+            )
+    ),
+    ISP(
+            "GeoIP2-ISP",
+            EnumSet.of(
+                    Field.IP,
+                    Field.AUTONOMOUS_SYSTEM_NUMBER,
+                    Field.AUTONOMOUS_SYSTEM_ORGANIZATION,
+                    Field.ISP,
+                    Field.ORGANIZATION
+            )
+    ),
+    ANONYMOUS_IP(
+            "GeoIP2-Anonymous-IP",
+            EnumSet.of(
+                    Field.HOSTING_PROVIDER,
+                    Field.TOR_EXIT_NODE,
+                    Field.ANONYMOUS_VPN,
+                    Field.ANONYMOUS,
+                    Field.PUBLIC_PROXY,
+                    Field.RESIDENTIAL_PROXY
+            )
+    ),
+    ENTERPRISE(
+            "Enterprise",
+            EnumSet.of(
+                    Field.IP,
+                    Field.COUNTRY_CODE2,
+                    Field.COUNTRY_NAME,
+                    Field.CONTINENT_NAME,
+                    Field.REGION_ISO_CODE,
+                    Field.REGION_NAME,
+                    Field.CITY_NAME,
+                    Field.LOCATION
+            )
+    ),
+    UNKNOWN(
+            "Unknown",
+            EnumSet.noneOf(Field.class)
+    );
+
+    private final String databaseType;
+    private final Set<Field> defaultFields;
+
+    Database(String databaseType, final Set<Field> defaultFields) {
+        this.databaseType = databaseType;
+        this.defaultFields = defaultFields;
+    }
+
+    public Set<Field> getDefaultFields() {
+        return Collections.unmodifiableSet(defaultFields);
+    }
+
+    public static Database fromDatabaseType(final String type) {
+        // It follows the same com.maxmind.geoip2.DatabaseReader#getDatabaseType logic
+        if (type.contains(CITY.databaseType)) {
+            return Database.CITY;
+        } else if (type.contains(COUNTRY.databaseType)) {
+            return Database.COUNTRY;
+        } else if (type.contains(DOMAIN.databaseType)) {
+            return Database.DOMAIN;
+        } else if (type.contains(ASN.databaseType)) {
+            return Database.ASN;
+        } else if (type.contains(ISP.databaseType)) {
+            return Database.ISP;
+        } else if (type.contains(ENTERPRISE.databaseType)) {
+            return Database.ENTERPRISE;
+        } else if (type.contains(ANONYMOUS_IP.databaseType)) {
+            return Database.ANONYMOUS_IP;
+        }
+
+        // The reason why we have this UNKNOWN type here, is to keep it backward compatible,
+        // allowing the pipeline to start, even if the plugin is configured to a non supported
+        // database type.
+        return Database.UNKNOWN;
+    }
+}