diff --git a/CHANGELOG.md b/CHANGELOG.md index cad098a..099f5e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 7.3.0 + - Added support for MaxMind GeoIP2 Enterprise and Anonymous-IP databases ([#223](https://github.com/logstash-plugins/logstash-filter-geoip/pull/223)) + - Updated MaxMind dependencies. + - Added tests for the Java classes. + ## 7.2.13 - [DOC] Add documentation for database auto-update configuration [#210](https://github.com/logstash-plugins/logstash-filter-geoip/pull/210) diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..8b23b8d --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +7.3.0 \ No newline at end of file diff --git a/build.gradle b/build.gradle index 0ce51a6..aba7780 100644 --- a/build.gradle +++ b/build.gradle @@ -22,11 +22,17 @@ apply plugin: "java" apply plugin: "distribution" apply plugin: "idea" -// TODO(sissel): Move this to a file shared by the gemspec. group "org.logstash.filters" -version "6.0.0" +version "${new File("VERSION").text.trim()}" project.archivesBaseName = "logstash-filter-geoip" +String junitVersion = '5.9.2' +String maxmindGeoip2Version = '2.17.0' +String maxmindDbVersion = '2.1.0' +String log4jVersion = '2.17.1' +String jrubyCompleteVersion = '9.1.13.0' +String mockitoVersion = '4.11.0' + sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 @@ -53,29 +59,38 @@ configurations { } dependencies { - compileOnly group: "org.apache.logging.log4j", name: "log4j-api", version: "2.17.1" - compileOnly group: "org.apache.logging.log4j", name: "log4j-core", version: "2.17.1" - compileOnly group: "com.maxmind.geoip2", name: "geoip2", version: "2.9.0" - compileOnly group: "com.maxmind.db", name: "maxmind-db", version: "1.2.2" - compileOnly group: 'org.jruby', name: 'jruby-complete', version: "1.7.26" + compileOnly group: "org.apache.logging.log4j", name: "log4j-api", version: log4jVersion + compileOnly group: "org.apache.logging.log4j", name: "log4j-core", version: log4jVersion + compileOnly group: 'org.jruby', name: 'jruby-complete', version: jrubyCompleteVersion compileOnly fileTree(dir: logstashCoreGemPath, include: '**/*.jar') - runtimeOnly group: "com.maxmind.geoip2", name: "geoip2", version: "2.9.0" - runtimeOnly group: "com.maxmind.db", name: "maxmind-db", version: "1.2.2" + implementation group: "com.maxmind.geoip2", name: "geoip2", version: maxmindGeoip2Version + implementation group: "com.maxmind.db", name: "maxmind-db", version: maxmindDbVersion - testImplementation group: 'junit', name: 'junit', version: '4.12' - testImplementation group: "org.apache.logging.log4j", name: "log4j-api", version: "2.17.1" - testImplementation group: "org.apache.logging.log4j", name: "log4j-core", version: "2.17.1" - testImplementation group: 'org.jruby', name: 'jruby-complete', version: "1.7.26" - testImplementation group: "com.maxmind.geoip2", name: "geoip2", version: "2.9.0" - testImplementation group: "com.maxmind.db", name: "maxmind-db", version: "1.2.2" - testImplementation fileTree(dir: logstashCoreGemPath, include: '**/*.jar') + testImplementation group: "org.junit.jupiter", name: "junit-jupiter-api", version: junitVersion + testImplementation group: "org.junit.jupiter", name: "junit-jupiter-params", version: junitVersion + testRuntimeOnly group: "org.junit.jupiter", name: "junit-jupiter-engine", version: junitVersion + testImplementation group: "org.apache.logging.log4j", name: "log4j-api", version: log4jVersion + testImplementation group: 'org.jruby', name: 'jruby-complete', version: jrubyCompleteVersion + testImplementation group: "org.mockito", name: "mockito-core", version: mockitoVersion geolite2('org.elasticsearch:geolite2-databases:20191119') { transitive = false } } +test { + useJUnitPlatform() + + testLogging { + events "started", "passed", "skipped", "failed", "standard_out", "standard_error" + } +} + +configurations { + testImplementation.extendsFrom compileOnly +} + task generateGemJarRequiresFile { doLast { File jars_file = file("lib/${project.archivesBaseName}_jars.rb") diff --git a/docs/index.asciidoc b/docs/index.asciidoc index e6cd508..988e0a5 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -190,37 +190,44 @@ When ECS compatibility is enabled, the fields are structured to fit into an ECS |=========================== | Database Field Name | ECS Field | Example -| `ip` | `[ip]` | `12.34.56.78` - -| `city_name` | `[geo][city_name]` | `Seattle` -| `country_name` | `[geo][country_name]` | `United States` -| `continent_code` | `[geo][continent_code]` | `NA` -| `continent_name` | `[geo][continent_name]` | `North America` -| `country_code2` | `[geo][country_iso_code]` | `US` -| `country_code3` | _N/A_ | `US` - - _maintained for legacy - support, but populated - with 2-character country - code_ - -| `postal_code` | `[geo][postal_code]` | `98106` -| `region_name` | `[geo][region_name]` | `Washington` -| `region_code` | `[geo][region_code]` | `WA` -| `region_iso_code`* | `[geo][region_iso_code]` | `US-WA` -| `timezone` | `[geo][timezone]` | `America/Los_Angeles` -| `location`* | `[geo][location]` | `{"lat": 47.6062, "lon": -122.3321}"` -| `latitude` | `[geo][location][lat]` | `47.6062` -| `longitude` | `[geo][location][lon]` | `-122.3321` - -| `domain` | `[domain]` | `example.com` - -| `asn` | `[as][number]` | `98765` -| `as_org` | `[as][organization][name]` | `Elastic, NV` - -| `isp` | `[mmdb][isp]` | `InterLink Supra LLC` -| `dma_code` | `[mmdb][dma_code]` | `819` -| `organization` | `[mmdb][organization]` | `Elastic, NV` +| `ip` | `[ip]` | `12.34.56.78` +| `anonymous` | `[ip_traits][anonymous]` | `false` +| `anonymous_vpn` | `[ip_traits][anonymous_vpn]` | `false` +| `hosting_provider` | `[ip_traits][hosting_provider]` | `true` +| `network` | `[ip_traits][network]` | `12.34.56.78/20` +| `public_proxy` | `[ip_traits][public_proxy]` | `true` +| `residential_proxy` | `[ip_traits][residential_proxy]` | `false` +| `tor_exit_node` | `[ip_traits][tor_exit_node]` | `true` + +| `city_name` | `[geo][city_name]` | `Seattle` +| `country_name` | `[geo][country_name]` | `United States` +| `continent_code` | `[geo][continent_code]` | `NA` +| `continent_name` | `[geo][continent_name]` | `North America` +| `country_code2` | `[geo][country_iso_code]` | `US` +| `country_code3` | _N/A_ | `US` + + _maintained for legacy + support, but populated + with 2-character country + code_ + +| `postal_code` | `[geo][postal_code]` | `98106` +| `region_name` | `[geo][region_name]` | `Washington` +| `region_code` | `[geo][region_code]` | `WA` +| `region_iso_code`* | `[geo][region_iso_code]` | `US-WA` +| `timezone` | `[geo][timezone]` | `America/Los_Angeles` +| `location`* | `[geo][location]` | `{"lat": 47.6062, "lon": -122.3321}"` +| `latitude` | `[geo][location][lat]` | `47.6062` +| `longitude` | `[geo][location][lon]` | `-122.3321` + +| `domain` | `[domain]` | `example.com` + +| `asn` | `[as][number]` | `98765` +| `as_org` | `[as][organization][name]` | `Elastic, NV` + +| `isp` | `[mmdb][isp]` | `InterLink Supra LLC` +| `dma_code` | `[mmdb][dma_code]` | `819` +| `organization` | `[mmdb][organization]` | `Elastic, NV` |=========================== NOTE: `*` indicates a composite field, which is only populated if GeoIP lookup result contains all components. @@ -301,7 +308,7 @@ number of cache misses and waste memory. The path to MaxMind's database file that Logstash should use. The default database is `GeoLite2-City`. This plugin supports several free databases (`GeoLite2-City`, `GeoLite2-Country`, `GeoLite2-ASN`) -and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`). +and a selection of commercially-licensed databases (`GeoIP2-City`, `GeoIP2-ISP`, `GeoIP2-Country`, `GeoIP2-Domain`, `GeoIP2-Enterprise`, `GeoIP2-Anonymous-IP`). Database auto-update applies to the default distribution. When `database` points to user's database path, auto-update is disabled. diff --git a/lib/logstash-filter-geoip_jars.rb b/lib/logstash-filter-geoip_jars.rb index fa0c77e..abdc729 100644 --- a/lib/logstash-filter-geoip_jars.rb +++ b/lib/logstash-filter-geoip_jars.rb @@ -1,6 +1,6 @@ # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT. require 'jar_dependencies' -require_jar('com.maxmind.geoip2', 'geoip2', '2.9.0') -require_jar('com.maxmind.db', 'maxmind-db', '1.2.2') -require_jar('org.logstash.filters', 'logstash-filter-geoip', '6.0.0') +require_jar('com.maxmind.geoip2', 'geoip2', '2.17.0') +require_jar('com.maxmind.db', 'maxmind-db', '2.1.0') +require_jar('org.logstash.filters', 'logstash-filter-geoip', '7.3.0') diff --git a/lib/logstash/filters/geoip.rb b/lib/logstash/filters/geoip.rb index 6b702c5..1dd742f 100644 --- a/lib/logstash/filters/geoip.rb +++ b/lib/logstash/filters/geoip.rb @@ -171,7 +171,17 @@ def fail_filter end def close - @database_manager.unsubscribe_database_path(@default_database_type, self) if @database_manager + begin + @database_manager.unsubscribe_database_path(@default_database_type, self) if @database_manager + rescue => e + @logger.error("Error unsubscribing geoip database path", :path => @database, :exception => e) + end + + begin + @geoipfilter.close if @geoipfilter + rescue => e + @logger.error("Error closing GeoIPFilter", :exception => e) + end end def select_database_path diff --git a/logstash-filter-geoip.gemspec b/logstash-filter-geoip.gemspec index b20ca7a..d1f9840 100644 --- a/logstash-filter-geoip.gemspec +++ b/logstash-filter-geoip.gemspec @@ -1,7 +1,9 @@ +VERSION = File.read(File.expand_path(File.join(File.dirname(__FILE__), "VERSION"))).strip unless defined?(VERSION) + Gem::Specification.new do |s| s.name = 'logstash-filter-geoip' - s.version = '7.2.13' + s.version = VERSION s.licenses = ['Apache License (2.0)'] s.summary = "Adds geographical information about an IP address" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/src/main/java/org/logstash/filters/geoip/Database.java b/src/main/java/org/logstash/filters/geoip/Database.java new file mode 100644 index 0000000..fc79219 --- /dev/null +++ b/src/main/java/org/logstash/filters/geoip/Database.java @@ -0,0 +1,125 @@ +package org.logstash.filters.geoip; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +enum Database { + + CITY( + "City", + EnumSet.of( + Field.IP, + Field.CITY_NAME, + Field.CONTINENT_CODE, + Field.COUNTRY_NAME, + Field.COUNTRY_CODE2, + Field.COUNTRY_CODE3, + Field.POSTAL_CODE, + Field.DMA_CODE, + Field.REGION_NAME, + Field.REGION_ISO_CODE, + Field.TIMEZONE, + Field.LOCATION, + Field.LATITUDE, + Field.LONGITUDE + ) + ), + COUNTRY( + "Country", + EnumSet.of( + Field.IP, + Field.COUNTRY_CODE2, + Field.COUNTRY_NAME, + Field.CONTINENT_NAME + ) + ), + DOMAIN( + "GeoIP2-Domain", + EnumSet.of( + Field.DOMAIN + ) + ), + ASN( + "GeoLite2-ASN", + EnumSet.of( + Field.IP, + Field.AUTONOMOUS_SYSTEM_NUMBER, + Field.AUTONOMOUS_SYSTEM_ORGANIZATION + ) + ), + ISP( + "GeoIP2-ISP", + EnumSet.of( + Field.IP, + Field.AUTONOMOUS_SYSTEM_NUMBER, + Field.AUTONOMOUS_SYSTEM_ORGANIZATION, + Field.ISP, + Field.ORGANIZATION + ) + ), + ANONYMOUS_IP( + "GeoIP2-Anonymous-IP", + EnumSet.of( + Field.HOSTING_PROVIDER, + Field.TOR_EXIT_NODE, + Field.ANONYMOUS_VPN, + Field.ANONYMOUS, + Field.PUBLIC_PROXY, + Field.RESIDENTIAL_PROXY + ) + ), + ENTERPRISE( + "Enterprise", + EnumSet.of( + Field.IP, + Field.COUNTRY_CODE2, + Field.COUNTRY_NAME, + Field.CONTINENT_NAME, + Field.REGION_ISO_CODE, + Field.REGION_NAME, + Field.CITY_NAME, + Field.LOCATION + ) + ), + UNKNOWN( + "Unknown", + EnumSet.noneOf(Field.class) + ); + + private final String databaseType; + private final Set defaultFields; + + Database(String databaseType, final Set defaultFields) { + this.databaseType = databaseType; + this.defaultFields = defaultFields; + } + + public Set getDefaultFields() { + return Collections.unmodifiableSet(defaultFields); + } + + public static Database fromDatabaseType(final String type) { + // It follows the same com.maxmind.geoip2.DatabaseReader#getDatabaseType logic + if (type.contains(CITY.databaseType)) { + return Database.CITY; + } else if (type.contains(COUNTRY.databaseType)) { + return Database.COUNTRY; + } else if (type.contains(DOMAIN.databaseType)) { + return Database.DOMAIN; + } else if (type.contains(ASN.databaseType)) { + return Database.ASN; + } else if (type.contains(ISP.databaseType)) { + return Database.ISP; + } else if (type.contains(ENTERPRISE.databaseType)) { + return Database.ENTERPRISE; + } else if (type.contains(ANONYMOUS_IP.databaseType)) { + return Database.ANONYMOUS_IP; + } + + // The reason why we have this UNKNOWN type here, is to keep it backward compatible, + // allowing the pipeline to start, even if the plugin is configured to a non supported + // database type. + return Database.UNKNOWN; + } +} \ No newline at end of file diff --git a/src/main/java/org/logstash/filters/geoip/Fields.java b/src/main/java/org/logstash/filters/geoip/Field.java similarity index 58% rename from src/main/java/org/logstash/filters/geoip/Fields.java rename to src/main/java/org/logstash/filters/geoip/Field.java index 87d9e0f..01d7375 100644 --- a/src/main/java/org/logstash/filters/geoip/Fields.java +++ b/src/main/java/org/logstash/filters/geoip/Field.java @@ -23,7 +23,8 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -enum Fields { +enum Field { + AUTONOMOUS_SYSTEM_NUMBER("as.number", "asn"), AUTONOMOUS_SYSTEM_ORGANIZATION("as.organization.name", "as_org"), CITY_NAME("geo.city_name", "city_name"), @@ -44,7 +45,14 @@ enum Fields { LOCATION("geo.location", "location"), LATITUDE("geo.location.lat", "latitude"), LONGITUDE("geo.location.lon", "longitude"), - ORGANIZATION("mmdb.organization", "organization"); + ORGANIZATION("mmdb.organization", "organization"), + NETWORK("ip_traits.network", "network"), + HOSTING_PROVIDER("ip_traits.hosting_provider", "hosting_provider"), + TOR_EXIT_NODE("ip_traits.tor_exit_node", "tor_exit_node"), + ANONYMOUS_VPN("ip_traits.anonymous_vpn", "anonymous_vpn"), + ANONYMOUS("ip_traits.anonymous", "anonymous"), + PUBLIC_PROXY("ip_traits.public_proxy", "public_proxy"), + RESIDENTIAL_PROXY("ip_traits.residential_proxy", "residential_proxy"); private final String fieldName; private final String ecsFieldName; @@ -53,11 +61,11 @@ enum Fields { private final String fieldReferenceECSv1; @Deprecated - Fields(String fieldName) { + Field(String fieldName) { this(fieldName, fieldName); } - Fields(final String ecsFieldName, final String legacyFieldName) { + Field(final String ecsFieldName, final String legacyFieldName) { this.ecsFieldName = ecsFieldName; this.fieldName = legacyFieldName; @@ -81,56 +89,22 @@ public String getFieldReferenceECSv1() { return this.fieldReferenceECSv1; } - private static final Map MAPPING; - static { - final Map mapping = new HashMap<>(); - for (Fields value : values()) { - mapping.put(value.name().toUpperCase(Locale.ROOT), value); - } - MAPPING = Collections.unmodifiableMap(mapping); - } - - static final EnumSet ALL_FIELDS = EnumSet.allOf(Fields.class); - - static final EnumSet DEFAULT_CITY_FIELDS = EnumSet.of(Fields.IP, Fields.CITY_NAME, - Fields.CONTINENT_CODE, Fields.COUNTRY_NAME, Fields.COUNTRY_CODE2, - Fields.COUNTRY_CODE3, Fields.IP, Fields.POSTAL_CODE, Fields.DMA_CODE, Fields.REGION_NAME, - Fields.REGION_CODE, Fields.TIMEZONE, Fields.LOCATION, Fields.LATITUDE, Fields.LONGITUDE); - - // When ECS is enabled, the composite REGION_ISO_CODE field is preferred to separate REGION_CODE - static final EnumSet DEFAULT_ECS_CITY_FIELDS; - static { - DEFAULT_ECS_CITY_FIELDS = EnumSet.copyOf(DEFAULT_CITY_FIELDS); - DEFAULT_ECS_CITY_FIELDS.remove(REGION_CODE); - DEFAULT_ECS_CITY_FIELDS.add(REGION_ISO_CODE); - } - - static final EnumSet DEFAULT_COUNTRY_FIELDS = EnumSet.of(Fields.IP, Fields.COUNTRY_CODE2, - Fields.IP, Fields.COUNTRY_NAME, Fields.CONTINENT_NAME); - - static final EnumSet DEFAULT_ISP_FIELDS = EnumSet.of(Fields.IP, Fields.AUTONOMOUS_SYSTEM_NUMBER, - Fields.AUTONOMOUS_SYSTEM_ORGANIZATION, Fields.ISP, Fields.ORGANIZATION); - - static final EnumSet DEFAULT_ASN_LITE_FIELDS = EnumSet.of(Fields.IP, Fields.AUTONOMOUS_SYSTEM_NUMBER, - Fields.AUTONOMOUS_SYSTEM_ORGANIZATION); - - static final EnumSet DEFAULT_DOMAIN_FIELDS = EnumSet.of(Fields.DOMAIN); - - public static Fields parseField(String value) { - final Fields fields = MAPPING.get(value.toUpperCase(Locale.ROOT)); - if (fields == null) { + public static Field parseField(String value) { + final String candidate = value.toUpperCase(Locale.ROOT); + try { + return Field.valueOf(candidate); + } catch (IllegalArgumentException e) { throw new IllegalArgumentException("illegal field value " + value + ". valid values are " + - Arrays.toString(ALL_FIELDS.toArray())); + Arrays.toString(Field.values())); } - return fields; } /** * Normalizes a dot-separated field path into a bracket-notation Logstash Field Reference * @param fieldName: a dot-separated field path (e.g., `geo.location.lat`) - * @return: a bracket-notation Field Reference (e.g., `[geo][location][lat]`) + * @return a bracket-notation Field Reference (e.g., `[geo][location][lat]`) */ - private static String normalizeFieldReferenceFragment(final String fieldName) { + static String normalizeFieldReferenceFragment(final String fieldName) { return Stream.of(fieldName.split("\\.")) .map((f) -> "[" + f + "]") .collect(Collectors.joining()); diff --git a/src/main/java/org/logstash/filters/geoip/GeoIPFilter.java b/src/main/java/org/logstash/filters/geoip/GeoIPFilter.java index 68b00f5..27c9185 100644 --- a/src/main/java/org/logstash/filters/geoip/GeoIPFilter.java +++ b/src/main/java/org/logstash/filters/geoip/GeoIPFilter.java @@ -20,12 +20,15 @@ import com.maxmind.db.CHMCache; import com.maxmind.db.InvalidDatabaseException; +import com.maxmind.db.Network; import com.maxmind.geoip2.exception.AddressNotFoundException; import com.maxmind.geoip2.exception.GeoIp2Exception; +import com.maxmind.geoip2.model.AnonymousIpResponse; import com.maxmind.geoip2.model.AsnResponse; import com.maxmind.geoip2.model.CityResponse; import com.maxmind.geoip2.model.CountryResponse; import com.maxmind.geoip2.model.DomainResponse; +import com.maxmind.geoip2.model.EnterpriseResponse; import com.maxmind.geoip2.model.IspResponse; import com.maxmind.geoip2.record.*; import org.apache.logging.log4j.LogManager; @@ -35,37 +38,23 @@ import com.maxmind.geoip2.DatabaseReader; import org.logstash.ext.JrubyEventExtLibrary.RubyEvent; +import java.io.Closeable; import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.*; import java.util.function.Function; +import java.util.stream.Collectors; -public class GeoIPFilter { - private static Logger logger = LogManager.getLogger(); - // The free GeoIP2 databases - private static final String CITY_LITE_DB_TYPE = "GeoLite2-City"; - private static final String COUNTRY_LITE_DB_TYPE = "GeoLite2-Country"; - private static final String ASN_LITE_DB_TYPE = "GeoLite2-ASN"; - - // The paid GeoIP2 databases - private static final String CITY_DB_TYPE = "GeoIP2-City"; - private static final String CITY_AFRICA_DB_TYPE = "GeoIP2-City-Africa"; - private static final String CITY_ASIA_PACIFIC_DB_TYPE = "GeoIP2-City-Asia-Pacific"; - private static final String CITY_EUROPE_DB_TYPE = "GeoIP2-City-Europe"; - private static final String CITY_NORTH_AMERICA_DB_TYPE = "GeoIP2-City-North-America"; - private static final String CITY_SOUTH_AMERICA_DB_TYPE = "GeoIP2-City-South-America"; - private static final String COUNTRY_DB_TYPE = "GeoIP2-Country"; - private static final String ISP_DB_TYPE = "GeoIP2-ISP"; - private static final String DOMAIN_DB_TYPE = "GeoIP2-Domain"; - +public class GeoIPFilter implements Closeable { + private static final Logger logger = LogManager.getLogger(); private final String sourceField; private final String targetField; - private final Set desiredFields; + private final Set desiredFields; + private final Database database; private final DatabaseReader databaseReader; - - private final Function fieldReferenceExtractor; + private final Function fieldReferenceExtractor; public GeoIPFilter(String sourceField, String targetField, List fields, String databasePath, int cacheSize, String ecsCompatibility) { @@ -73,31 +62,43 @@ public GeoIPFilter(String sourceField, String targetField, List fields, this.targetField = targetField; switch (ecsCompatibility) { case "disabled": - this.fieldReferenceExtractor = Fields::getFieldReferenceLegacy; + this.fieldReferenceExtractor = Field::getFieldReferenceLegacy; break; case "v1": case "v8": - this.fieldReferenceExtractor = Fields::getFieldReferenceECSv1; + this.fieldReferenceExtractor = Field::getFieldReferenceECSv1; break; default: throw new UnsupportedOperationException("Unknown ECS version " + ecsCompatibility); } - final File database = new File(databasePath); + final File databaseFile = new File(databasePath); try { - this.databaseReader = new DatabaseReader.Builder(database).withCache(new CHMCache(cacheSize)).build(); + this.databaseReader = new DatabaseReader.Builder(databaseFile).withCache(new CHMCache(cacheSize)).build(); } catch (InvalidDatabaseException e) { throw new IllegalArgumentException("The database provided is invalid or corrupted.", e); } catch (IOException e) { throw new IllegalArgumentException("The database provided was not found in the path", e); } + + this.database = getDatabase(databaseReader); this.desiredFields = createDesiredFields(fields, !ecsCompatibility.equals("disabled")); } + private static Database getDatabase(DatabaseReader reader) { + final String databaseType = reader.getMetadata().getDatabaseType(); + final Database database = Database.fromDatabaseType(databaseType); + + if (database == Database.UNKNOWN) { + logger.warn("The provided database type {} is not supported", databaseType); + } + + return database; + } + public static boolean isDatabaseValid(String databasePath) { final File database = new File(databasePath); - try { - new DatabaseReader.Builder(database).build(); + try (DatabaseReader ignore = new DatabaseReader.Builder(database).build()) { return true; } catch (InvalidDatabaseException e) { logger.debug("The database provided is invalid or corrupted"); @@ -107,38 +108,31 @@ public static boolean isDatabaseValid(String databasePath) { return false; } - private Set createDesiredFields(List fields, final boolean ecsCompatibilityEnabled) { - Set desiredFields = EnumSet.noneOf(Fields.class); - if (fields == null || fields.isEmpty()) { - switch (databaseReader.getMetadata().getDatabaseType()) { - case CITY_LITE_DB_TYPE: - case CITY_DB_TYPE: - case CITY_AFRICA_DB_TYPE: - case CITY_ASIA_PACIFIC_DB_TYPE: - case CITY_EUROPE_DB_TYPE: - case CITY_NORTH_AMERICA_DB_TYPE: - case CITY_SOUTH_AMERICA_DB_TYPE: - desiredFields = ecsCompatibilityEnabled ? Fields.DEFAULT_ECS_CITY_FIELDS : Fields.DEFAULT_CITY_FIELDS; - break; - case COUNTRY_LITE_DB_TYPE: - case COUNTRY_DB_TYPE: - desiredFields = Fields.DEFAULT_COUNTRY_FIELDS; - break; - case ISP_DB_TYPE: - desiredFields = Fields.DEFAULT_ISP_FIELDS; - break; - case ASN_LITE_DB_TYPE: - desiredFields = Fields.DEFAULT_ASN_LITE_FIELDS; - break; - case DOMAIN_DB_TYPE: - desiredFields = Fields.DEFAULT_DOMAIN_FIELDS; - } - } else { - for (String fieldName : fields) { - desiredFields.add(Fields.parseField(fieldName)); - } + private Set createDesiredFields(List fields, final boolean ecsCompatibilityEnabled) { + if (fields != null && !fields.isEmpty()) { + return fields.stream() + .map(Field::parseField) + .collect(Collectors.toCollection(() -> EnumSet.noneOf(Field.class))); + } + + if (database == Database.CITY) { + return createCityDefaultFields(ecsCompatibilityEnabled); } - return desiredFields; + + return database.getDefaultFields(); + } + + private Set createCityDefaultFields(boolean ecsCompatibilityEnabled) { + // When ECS is disabled, change the default region code field from REGION_ISO_CODE to + // REGION_CODE (BC) + if (!ecsCompatibilityEnabled) { + final EnumSet ecsDisabledFields = EnumSet.copyOf(database.getDefaultFields()); + ecsDisabledFields.remove(Field.REGION_ISO_CODE); + ecsDisabledFields.add(Field.REGION_CODE); + return ecsDisabledFields; + } + + return database.getDefaultFields(); } public boolean handleEvent(RubyEvent rubyEvent) { @@ -162,33 +156,32 @@ public boolean handleEvent(RubyEvent rubyEvent) { return false; } - Map geoData = new HashMap<>(); + Map geoData = new HashMap<>(); try { final InetAddress ipAddress = InetAddress.getByName(ip); - switch (databaseReader.getMetadata().getDatabaseType()) { - case CITY_LITE_DB_TYPE: - case CITY_DB_TYPE: - case CITY_AFRICA_DB_TYPE: - case CITY_ASIA_PACIFIC_DB_TYPE: - case CITY_EUROPE_DB_TYPE: - case CITY_NORTH_AMERICA_DB_TYPE: - case CITY_SOUTH_AMERICA_DB_TYPE: + switch (database) { + case CITY: geoData = retrieveCityGeoData(ipAddress); break; - case COUNTRY_LITE_DB_TYPE: - case COUNTRY_DB_TYPE: + case COUNTRY: geoData = retrieveCountryGeoData(ipAddress); break; - case ASN_LITE_DB_TYPE: + case ASN: geoData = retrieveAsnGeoData(ipAddress); break; - case ISP_DB_TYPE: + case ISP: geoData = retrieveIspGeoData(ipAddress); break; - case DOMAIN_DB_TYPE: + case DOMAIN: geoData = retrieveDomainGeoData(ipAddress); break; + case ENTERPRISE: + geoData = retrieveEnterpriseGeoData(ipAddress); + break; + case ANONYMOUS_IP: + geoData = retrieveAnonymousIpGeoData(ipAddress); + break; default: throw new IllegalStateException("Unsupported database type " + databaseReader.getMetadata().getDatabaseType() + ""); } @@ -203,7 +196,7 @@ public boolean handleEvent(RubyEvent rubyEvent) { return applyGeoData(geoData, event); } - private boolean applyGeoData(Map geoData, Event event) { + private boolean applyGeoData(Map geoData, Event event) { if (geoData == null) { return false; } @@ -217,8 +210,8 @@ private boolean applyGeoData(Map geoData, Event event) { } String targetFieldReference = "[" + this.targetField + "]"; - for (Map.Entry it: geoData.entrySet()) { - final Fields field = it.getKey(); + for (Map.Entry it: geoData.entrySet()) { + final Field field = it.getKey(); final String subFieldReference = this.fieldReferenceExtractor.apply(field); if (subFieldReference.equals("[]")) { @@ -230,7 +223,7 @@ private boolean applyGeoData(Map geoData, Event event) { return true; } - private Map retrieveCityGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { + private Map retrieveCityGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { CityResponse response = databaseReader.city(ipAddress); Country country = response.getCountry(); City city = response.getCity(); @@ -238,7 +231,7 @@ private Map retrieveCityGeoData(InetAddress ipAddress) throws Geo Continent continent = response.getContinent(); Postal postal = response.getPostal(); Subdivision subdivision = response.getMostSpecificSubdivision(); - Map geoData = new HashMap<>(); + Map geoData = new EnumMap<>(Field.class); // if location is empty, there is no point populating geo data // and most likely all other fields are empty as well @@ -246,104 +239,95 @@ private Map retrieveCityGeoData(InetAddress ipAddress) throws Geo return geoData; } - for (Fields desiredField : this.desiredFields) { + for (Field desiredField : this.desiredFields) { switch (desiredField) { case CITY_NAME: String cityName = city.getName(); if (cityName != null) { - geoData.put(Fields.CITY_NAME, cityName); + geoData.put(Field.CITY_NAME, cityName); } break; case CONTINENT_CODE: String continentCode = continent.getCode(); if (continentCode != null) { - geoData.put(Fields.CONTINENT_CODE, continentCode); + geoData.put(Field.CONTINENT_CODE, continentCode); } break; case CONTINENT_NAME: String continentName = continent.getName(); if (continentName != null) { - geoData.put(Fields.CONTINENT_NAME, continentName); + geoData.put(Field.CONTINENT_NAME, continentName); } break; case COUNTRY_NAME: String countryName = country.getName(); if (countryName != null) { - geoData.put(Fields.COUNTRY_NAME, countryName); + geoData.put(Field.COUNTRY_NAME, countryName); } break; case COUNTRY_CODE2: String countryCode2 = country.getIsoCode(); if (countryCode2 != null) { - geoData.put(Fields.COUNTRY_CODE2, countryCode2); + geoData.put(Field.COUNTRY_CODE2, countryCode2); } break; case COUNTRY_CODE3: String countryCode3 = country.getIsoCode(); if (countryCode3 != null) { - geoData.put(Fields.COUNTRY_CODE3, countryCode3); + geoData.put(Field.COUNTRY_CODE3, countryCode3); } break; case IP: - geoData.put(Fields.IP, ipAddress.getHostAddress()); + geoData.put(Field.IP, ipAddress.getHostAddress()); break; case POSTAL_CODE: String postalCode = postal.getCode(); if (postalCode != null) { - geoData.put(Fields.POSTAL_CODE, postalCode); + geoData.put(Field.POSTAL_CODE, postalCode); } break; case DMA_CODE: Integer dmaCode = location.getMetroCode(); if (dmaCode != null) { - geoData.put(Fields.DMA_CODE, dmaCode); + geoData.put(Field.DMA_CODE, dmaCode); } break; case REGION_NAME: String subdivisionName = subdivision.getName(); if (subdivisionName != null) { - geoData.put(Fields.REGION_NAME, subdivisionName); + geoData.put(Field.REGION_NAME, subdivisionName); } break; case REGION_CODE: String subdivisionCode = subdivision.getIsoCode(); if (subdivisionCode != null) { - geoData.put(Fields.REGION_CODE, subdivisionCode); + geoData.put(Field.REGION_CODE, subdivisionCode); } break; case REGION_ISO_CODE: - String countryCodeForRegion = country.getIsoCode(); - String regionCode2 = subdivision.getIsoCode(); - if (countryCodeForRegion != null && regionCode2 != null) { - geoData.put(Fields.REGION_ISO_CODE, String.format("%s-%s", countryCodeForRegion, regionCode2)); - } + parseRegionIsoCodeField(country, subdivision) + .ifPresent(data -> geoData.put(Field.REGION_ISO_CODE, data)); break; case TIMEZONE: String locationTimeZone = location.getTimeZone(); if (locationTimeZone != null) { - geoData.put(Fields.TIMEZONE, locationTimeZone); + geoData.put(Field.TIMEZONE, locationTimeZone); } break; case LOCATION: - Double latitude = location.getLatitude(); - Double longitude = location.getLongitude(); - if (latitude != null && longitude != null) { - Map locationObject = new HashMap<>(); - locationObject.put("lat", latitude); - locationObject.put("lon", longitude); - geoData.put(Fields.LOCATION, locationObject); - } + parseLocationField(location) + .ifPresent(data -> geoData.put(Field.LOCATION, data)); break; case LATITUDE: Double lat = location.getLatitude(); if (lat != null) { - geoData.put(Fields.LATITUDE, lat); + geoData.put(Field.LATITUDE, lat); } break; case LONGITUDE: Double lon = location.getLongitude(); if (lon != null) { - geoData.put(Fields.LONGITUDE, lon); + geoData.put(Field.LONGITUDE, lon); } break; } @@ -352,33 +336,33 @@ private Map retrieveCityGeoData(InetAddress ipAddress) throws Geo return geoData; } - private Map retrieveCountryGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { + private Map retrieveCountryGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { CountryResponse response = databaseReader.country(ipAddress); Country country = response.getCountry(); Continent continent = response.getContinent(); - Map geoData = new HashMap<>(); + Map geoData = new EnumMap<>(Field.class); - for (Fields desiredField : this.desiredFields) { + for (Field desiredField : this.desiredFields) { switch (desiredField) { case IP: - geoData.put(Fields.IP, ipAddress.getHostAddress()); + geoData.put(Field.IP, ipAddress.getHostAddress()); break; case COUNTRY_CODE2: String countryCode2 = country.getIsoCode(); if (countryCode2 != null) { - geoData.put(Fields.COUNTRY_CODE2, countryCode2); + geoData.put(Field.COUNTRY_CODE2, countryCode2); } break; case COUNTRY_NAME: String countryName = country.getName(); if (countryName != null) { - geoData.put(Fields.COUNTRY_NAME, countryName); + geoData.put(Field.COUNTRY_NAME, countryName); } break; case CONTINENT_NAME: String continentName = continent.getName(); if (continentName != null) { - geoData.put(Fields.CONTINENT_NAME, continentName); + geoData.put(Field.CONTINENT_NAME, continentName); } break; } @@ -387,37 +371,37 @@ private Map retrieveCountryGeoData(InetAddress ipAddress) throws return geoData; } - private Map retrieveIspGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { + private Map retrieveIspGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { IspResponse response = databaseReader.isp(ipAddress); - Map geoData = new HashMap<>(); - for (Fields desiredField : this.desiredFields) { + Map geoData = new EnumMap<>(Field.class); + for (Field desiredField : this.desiredFields) { switch (desiredField) { case IP: - geoData.put(Fields.IP, ipAddress.getHostAddress()); + geoData.put(Field.IP, ipAddress.getHostAddress()); break; case AUTONOMOUS_SYSTEM_NUMBER: Integer asn = response.getAutonomousSystemNumber(); if (asn != null) { - geoData.put(Fields.AUTONOMOUS_SYSTEM_NUMBER, asn); + geoData.put(desiredField, asn); } break; case AUTONOMOUS_SYSTEM_ORGANIZATION: String aso = response.getAutonomousSystemOrganization(); if (aso != null) { - geoData.put(Fields.AUTONOMOUS_SYSTEM_ORGANIZATION, aso); + geoData.put(desiredField, aso); } break; case ISP: String isp = response.getIsp(); if (isp != null) { - geoData.put(Fields.ISP, isp); + geoData.put(Field.ISP, isp); } break; case ORGANIZATION: String org = response.getOrganization(); if (org != null) { - geoData.put(Fields.ORGANIZATION, org); + geoData.put(Field.ORGANIZATION, org); } break; } @@ -426,24 +410,31 @@ private Map retrieveIspGeoData(InetAddress ipAddress) throws Geo return geoData; } - private Map retrieveAsnGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { + private Map retrieveAsnGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { AsnResponse response = databaseReader.asn(ipAddress); - Map geoData = new HashMap<>(); - for (Fields desiredField : this.desiredFields) { + Network network = response.getNetwork(); + + Map geoData = new EnumMap<>(Field.class); + for (Field desiredField : this.desiredFields) { switch (desiredField) { case IP: - geoData.put(Fields.IP, ipAddress.getHostAddress()); + geoData.put(Field.IP, ipAddress.getHostAddress()); break; case AUTONOMOUS_SYSTEM_NUMBER: Integer asn = response.getAutonomousSystemNumber(); if (asn != null) { - geoData.put(Fields.AUTONOMOUS_SYSTEM_NUMBER, asn); + geoData.put(Field.AUTONOMOUS_SYSTEM_NUMBER, asn); } break; case AUTONOMOUS_SYSTEM_ORGANIZATION: String aso = response.getAutonomousSystemOrganization(); if (aso != null) { - geoData.put(Fields.AUTONOMOUS_SYSTEM_ORGANIZATION, aso); + geoData.put(Field.AUTONOMOUS_SYSTEM_ORGANIZATION, aso); + } + break; + case NETWORK: + if (network != null) { + geoData.put(Field.NETWORK, network.toString()); } break; } @@ -452,18 +443,199 @@ private Map retrieveAsnGeoData(InetAddress ipAddress) throws Geo return geoData; } - private Map retrieveDomainGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { + private Map retrieveDomainGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { DomainResponse response = databaseReader.domain(ipAddress); - Map geoData = new HashMap<>(); - for (Fields desiredField : this.desiredFields) { + Map geoData = new EnumMap<>(Field.class); + for (Field desiredField : this.desiredFields) { switch (desiredField) { case DOMAIN: String domain = response.getDomain(); - geoData.put(Fields.DOMAIN, domain); + geoData.put(Field.DOMAIN, domain); break; } } return geoData; } + + private Map retrieveEnterpriseGeoData(InetAddress ipAddress) throws GeoIp2Exception, IOException { + EnterpriseResponse response = databaseReader.enterprise(ipAddress); + + Map geoData = new EnumMap<>(Field.class); + Country country = response.getCountry(); + City city = response.getCity(); + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + + Integer asn = response.getTraits().getAutonomousSystemNumber(); + String organizationName = response.getTraits().getAutonomousSystemOrganization(); + Network network = response.getTraits().getNetwork(); + + boolean isHostingProvider = response.getTraits().isHostingProvider(); + boolean isTorExitNode = response.getTraits().isTorExitNode(); + boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); + boolean isAnonymous = response.getTraits().isAnonymous(); + boolean isPublicProxy = response.getTraits().isPublicProxy(); + boolean isResidentialProxy = response.getTraits().isResidentialProxy(); + + for (Field desiredField : this.desiredFields) { + switch (desiredField) { + case IP: + geoData.put(Field.IP, ipAddress.getHostAddress()); + break; + case COUNTRY_CODE2: + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + geoData.put(desiredField, countryIsoCode); + } + break; + case COUNTRY_NAME: + String countryName = country.getName(); + if (countryName != null) { + geoData.put(desiredField, countryName); + } + break; + case CONTINENT_NAME: + String continentName = continent.getName(); + if (continentName != null) { + geoData.put(desiredField, continentName); + } + break; + case REGION_ISO_CODE: + parseRegionIsoCodeField(country, subdivision) + .ifPresent(data -> geoData.put(desiredField, data)); + break; + case REGION_NAME: + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + geoData.put(desiredField, subdivisionName); + } + break; + case CITY_NAME: + String cityName = city.getName(); + if (cityName != null) { + geoData.put(desiredField, cityName); + } + break; + case TIMEZONE: + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + geoData.put(desiredField, locationTimeZone); + } + break; + case LOCATION: + parseLocationField(location) + .ifPresent(data -> geoData.put(desiredField, data)); + break; + case AUTONOMOUS_SYSTEM_NUMBER: + if (asn != null) { + geoData.put(desiredField, asn); + } + break; + case AUTONOMOUS_SYSTEM_ORGANIZATION: + if (organizationName != null) { + geoData.put(desiredField, organizationName); + } + break; + case NETWORK: + if (network != null) { + geoData.put(desiredField, network.toString()); + } + break; + case HOSTING_PROVIDER: + geoData.put(desiredField, isHostingProvider); + break; + case TOR_EXIT_NODE: + geoData.put(desiredField, isTorExitNode); + break; + case ANONYMOUS_VPN: + geoData.put(desiredField, isAnonymousVpn); + break; + case ANONYMOUS: + geoData.put(desiredField, isAnonymous); + break; + case PUBLIC_PROXY: + geoData.put(desiredField, isPublicProxy); + break; + case RESIDENTIAL_PROXY: + geoData.put(desiredField, isResidentialProxy); + break; + } + } + return geoData; + } + + private Map retrieveAnonymousIpGeoData(final InetAddress ipAddress) throws GeoIp2Exception, IOException { + AnonymousIpResponse response = databaseReader.anonymousIp(ipAddress); + + Map geoData = new EnumMap<>(Field.class); + boolean isHostingProvider = response.isHostingProvider(); + boolean isTorExitNode = response.isTorExitNode(); + boolean isAnonymousVpn = response.isAnonymousVpn(); + boolean isAnonymous = response.isAnonymous(); + boolean isPublicProxy = response.isPublicProxy(); + boolean isResidentialProxy = response.isResidentialProxy(); + + for (Field desiredField : this.desiredFields) { + switch (desiredField) { + case IP: + geoData.put(desiredField, ipAddress.getHostAddress()); + break; + case HOSTING_PROVIDER: + geoData.put(desiredField, isHostingProvider); + break; + case TOR_EXIT_NODE: + geoData.put(desiredField, isTorExitNode); + break; + case ANONYMOUS_VPN: + geoData.put(desiredField, isAnonymousVpn); + break; + case ANONYMOUS: + geoData.put(desiredField, isAnonymous); + break; + case PUBLIC_PROXY: + geoData.put(desiredField, isPublicProxy); + break; + case RESIDENTIAL_PROXY: + geoData.put(desiredField, isResidentialProxy); + break; + } + } + return geoData; + } + + private Optional> parseLocationField(Location location) { + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + return Optional.of(locationObject); + } + + return Optional.empty(); + } + + private Optional parseRegionIsoCodeField(final Country country, final Subdivision subdivision) { + String countryCodeForRegion = country.getIsoCode(); + String regionCode2 = subdivision.getIsoCode(); + if (countryCodeForRegion != null && regionCode2 != null) { + return Optional.of(String.format("%s-%s", countryCodeForRegion, regionCode2)); + } + + return Optional.empty(); + } + + @Override + public void close() { + if (databaseReader != null) { + try { + databaseReader.close(); + } catch (IOException e) { + // Ignore + } + } + } } diff --git a/src/test/java/org/logstash/filters/geoip/DatabaseTest.java b/src/test/java/org/logstash/filters/geoip/DatabaseTest.java new file mode 100644 index 0000000..45540e4 --- /dev/null +++ b/src/test/java/org/logstash/filters/geoip/DatabaseTest.java @@ -0,0 +1,163 @@ +package org.logstash.filters.geoip; + +import com.maxmind.geoip2.DatabaseReader; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class DatabaseTest { + + private static final Map> DATABASES_TO_MAXMIND_FILES = new HashMap>() { + { + put(Database.CITY, Arrays.asList(MaxMindDatabases.GEOIP2_CITY, MaxMindDatabases.GEOLITE2_CITY)); + put(Database.COUNTRY, Arrays.asList(MaxMindDatabases.GEOIP2_COUNTRY, MaxMindDatabases.GEOLITE2_COUNTRY)); + put(Database.DOMAIN, Collections.singletonList(MaxMindDatabases.GEOIP2_DOMAIN)); + put(Database.ASN, Collections.singletonList(MaxMindDatabases.GEOLITE2_ASN)); + put(Database.ANONYMOUS_IP, Collections.singletonList(MaxMindDatabases.GEOIP2_ANONYMOUS_IP)); + put(Database.ISP, Collections.singletonList(MaxMindDatabases.GEOIP2_ISP)); + put(Database.ENTERPRISE, Collections.singletonList(MaxMindDatabases.GEOIP2_ENTERPRISE)); + put(Database.UNKNOWN, Collections.emptyList()); + } + }; + + @Test + void testCityDefaultFields() { + final EnumSet expectedFields = EnumSet.of( + Field.IP, + Field.CITY_NAME, + Field.CONTINENT_CODE, + Field.COUNTRY_NAME, + Field.COUNTRY_CODE2, + Field.COUNTRY_CODE3, + Field.POSTAL_CODE, + Field.DMA_CODE, + Field.REGION_NAME, + Field.REGION_ISO_CODE, + Field.TIMEZONE, + Field.LOCATION, + Field.LATITUDE, + Field.LONGITUDE + ); + + assertEquals(expectedFields, Database.CITY.getDefaultFields()); + } + + @Test + void testCountryDefaultFields() { + final EnumSet expectedFields = EnumSet.of( + Field.IP, + Field.COUNTRY_CODE2, + Field.COUNTRY_NAME, + Field.CONTINENT_NAME + ); + + assertEquals(expectedFields, Database.COUNTRY.getDefaultFields()); + } + + @Test + void testDomainDefaultFields() { + final EnumSet expectedFields = EnumSet.of(Field.DOMAIN); + + assertEquals(expectedFields, Database.DOMAIN.getDefaultFields()); + } + + @Test + void testAsnDefaultFields() { + final EnumSet expectedFields = EnumSet.of( + Field.IP, + Field.AUTONOMOUS_SYSTEM_NUMBER, + Field.AUTONOMOUS_SYSTEM_ORGANIZATION + ); + + assertEquals(expectedFields, Database.ASN.getDefaultFields()); + } + + @Test + void testIspDefaultFields() { + final EnumSet expectedFields = EnumSet.of( + Field.IP, + Field.AUTONOMOUS_SYSTEM_NUMBER, + Field.AUTONOMOUS_SYSTEM_ORGANIZATION, + Field.ISP, + Field.ORGANIZATION + ); + + assertEquals(expectedFields, Database.ISP.getDefaultFields()); + } + + @Test + void testAnonymousIpDefaultFields() { + final EnumSet expectedFields = EnumSet.of( + Field.HOSTING_PROVIDER, + Field.TOR_EXIT_NODE, + Field.ANONYMOUS_VPN, + Field.ANONYMOUS, + Field.PUBLIC_PROXY, + Field.RESIDENTIAL_PROXY + ); + + assertEquals(expectedFields, Database.ANONYMOUS_IP.getDefaultFields()); + } + + @Test + void testEnterpriseDefaultFields() { + final EnumSet expectedFields = EnumSet.of( + Field.IP, + Field.COUNTRY_CODE2, + Field.COUNTRY_NAME, + Field.CONTINENT_NAME, + Field.REGION_ISO_CODE, + Field.REGION_NAME, + Field.CITY_NAME, + Field.LOCATION + ); + + assertEquals(expectedFields, Database.ENTERPRISE.getDefaultFields()); + } + + @ParameterizedTest + @EnumSource(Database.class) + void fromDatabaseTypeWithMaxMindFilesShouldReturnDatabase(Database expectedDatabase) throws IOException { + for (final Path path : DATABASES_TO_MAXMIND_FILES.get(expectedDatabase)) { + try (final DatabaseReader reader = new DatabaseReader + .Builder(path.toFile()) + .build()) { + + final String fileDatabaseType = reader.getMetadata().getDatabaseType(); + final Database parseDatabase = Database.fromDatabaseType(fileDatabaseType); + final String message = String.format("File '%s' was parsed as %s database instead of %s", path, parseDatabase, expectedDatabase); + + assertEquals(expectedDatabase, parseDatabase, message); + } + } + } + + @Test + void fromDatabaseTypeWithKnownDatabaseTypesShouldReturnDatabase() { + assertEquals(Database.CITY, Database.fromDatabaseType("GeoLite2-City")); + assertEquals(Database.CITY, Database.fromDatabaseType("GeoIP2-City")); + assertEquals(Database.CITY, Database.fromDatabaseType("GeoIP2-City-Africa")); + assertEquals(Database.CITY, Database.fromDatabaseType("GeoIP2-City-Asia-Pacific")); + assertEquals(Database.CITY, Database.fromDatabaseType("GeoIP2-City-Europe")); + assertEquals(Database.CITY, Database.fromDatabaseType("GeoIP2-City-North-America")); + assertEquals(Database.CITY, Database.fromDatabaseType("GeoIP2-City-South-America")); + assertEquals(Database.COUNTRY, Database.fromDatabaseType("GeoLite2-Country")); + assertEquals(Database.COUNTRY, Database.fromDatabaseType("GeoIP2-Country")); + assertEquals(Database.DOMAIN, Database.fromDatabaseType("GeoIP2-Domain")); + assertEquals(Database.ASN, Database.fromDatabaseType("GeoLite2-ASN")); + assertEquals(Database.ISP, Database.fromDatabaseType("GeoIP2-ISP")); + assertEquals(Database.ANONYMOUS_IP, Database.fromDatabaseType("GeoIP2-Anonymous-IP")); + assertEquals(Database.ENTERPRISE, Database.fromDatabaseType("Enterprise")); + } +} \ No newline at end of file diff --git a/src/test/java/org/logstash/filters/geoip/FieldTest.java b/src/test/java/org/logstash/filters/geoip/FieldTest.java new file mode 100644 index 0000000..059947f --- /dev/null +++ b/src/test/java/org/logstash/filters/geoip/FieldTest.java @@ -0,0 +1,65 @@ +package org.logstash.filters.geoip; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class FieldTest { + + @ParameterizedTest + @EnumSource(Field.class) + void parseFieldWithValidFieldNameShouldReturnField(Field field) { + final String lowerCaseFieldName = field.name().toLowerCase(); + assertEquals(field, Field.parseField(lowerCaseFieldName)); + } + + @Test + void parseFieldWithInvalidFieldNameShouldThrown() { + final IllegalArgumentException thrown = assertThrows(IllegalArgumentException.class, () -> Field.parseField("foobar")); + assertTrue(thrown.getMessage().startsWith("illegal field value foobar. valid values are")); + } + + @Test + void testFieldNames() { + assertFieldNames(Field.AUTONOMOUS_SYSTEM_NUMBER, "as.number", "asn"); + assertFieldNames(Field.AUTONOMOUS_SYSTEM_ORGANIZATION, "as.organization.name", "as_org"); + assertFieldNames(Field.CITY_NAME, "geo.city_name", "city_name"); + assertFieldNames(Field.COUNTRY_NAME, "geo.country_name", "country_name"); + assertFieldNames(Field.CONTINENT_CODE, "geo.continent_code", "continent_code"); + assertFieldNames(Field.CONTINENT_NAME, "geo.continent_name", "continent_name"); + assertFieldNames(Field.COUNTRY_CODE2, "geo.country_iso_code", "country_code2"); + assertFieldNames(Field.COUNTRY_CODE3, "", "country_code3"); + assertFieldNames(Field.DOMAIN, "domain", "domain"); + assertFieldNames(Field.IP, "ip", "ip"); + assertFieldNames(Field.ISP, "mmdb.isp", "isp"); + assertFieldNames(Field.POSTAL_CODE, "geo.postal_code", "postal_code"); + assertFieldNames(Field.DMA_CODE, "mmdb.dma_code", "dma_code"); + assertFieldNames(Field.REGION_NAME, "geo.region_name", "region_name"); + assertFieldNames(Field.REGION_CODE, "geo.region_code", "region_code"); + assertFieldNames(Field.REGION_ISO_CODE, "geo.region_iso_code", "region_iso_code"); + assertFieldNames(Field.TIMEZONE, "geo.timezone", "timezone"); + assertFieldNames(Field.LOCATION, "geo.location", "location"); + assertFieldNames(Field.LATITUDE, "geo.location.lat", "latitude"); + assertFieldNames(Field.LONGITUDE, "geo.location.lon", "longitude"); + assertFieldNames(Field.ORGANIZATION, "mmdb.organization", "organization"); + assertFieldNames(Field.NETWORK, "ip_traits.network", "network"); + assertFieldNames(Field.HOSTING_PROVIDER, "ip_traits.hosting_provider", "hosting_provider"); + assertFieldNames(Field.TOR_EXIT_NODE, "ip_traits.tor_exit_node", "tor_exit_node"); + assertFieldNames(Field.ANONYMOUS_VPN, "ip_traits.anonymous_vpn", "anonymous_vpn"); + assertFieldNames(Field.ANONYMOUS, "ip_traits.anonymous", "anonymous"); + assertFieldNames(Field.PUBLIC_PROXY, "ip_traits.public_proxy", "public_proxy"); + assertFieldNames(Field.RESIDENTIAL_PROXY, "ip_traits.residential_proxy", "residential_proxy"); + } + + void assertFieldNames(Field field, String expectedEcsFieldName, String expectedFieldName) { + assertEquals(expectedEcsFieldName, field.getEcsFieldName()); + assertEquals(Field.normalizeFieldReferenceFragment(expectedEcsFieldName), field.getFieldReferenceECSv1()); + + assertEquals(expectedFieldName, field.fieldName()); + assertEquals(Field.normalizeFieldReferenceFragment(expectedFieldName), field.getFieldReferenceLegacy()); + } +} \ No newline at end of file diff --git a/src/test/java/org/logstash/filters/geoip/GeoIPFilterTest.java b/src/test/java/org/logstash/filters/geoip/GeoIPFilterTest.java new file mode 100644 index 0000000..98c834a --- /dev/null +++ b/src/test/java/org/logstash/filters/geoip/GeoIPFilterTest.java @@ -0,0 +1,314 @@ +package org.logstash.filters.geoip; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.logstash.Event; + +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.logstash.RubyUtil.RUBY; +import static org.logstash.ext.JrubyEventExtLibrary.RubyEvent; + + +class GeoIPFilterTest { + + private static final String SOURCE_FIELD = "ip"; + private static final String TARGET_FIELD = "data"; + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void handleEventWithGeoIp2CityDatabaseShouldProperlyCreateEvent(boolean ecsEnabled) { + final List supportedFields = Arrays.asList( + Field.IP, + Field.CITY_NAME, + Field.CONTINENT_CODE, + Field.CONTINENT_NAME, + Field.COUNTRY_NAME, + Field.COUNTRY_CODE2, + Field.COUNTRY_CODE3, + Field.POSTAL_CODE, + Field.DMA_CODE, + Field.REGION_NAME, + Field.REGION_CODE, + Field.REGION_ISO_CODE, + Field.TIMEZONE, + Field.LOCATION, + Field.LATITUDE, + Field.LONGITUDE + ); + + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOIP2_CITY, ecsEnabled, supportedFields)) { + final RubyEvent rubyEvent = createRubyEvent("216.160.83.58"); + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + assertEquals("216.160.83.58", getField(event, Field.IP, ecsEnabled)); + assertEquals("Milton", getField(event, Field.CITY_NAME, ecsEnabled)); + assertEquals("NA", getField(event, Field.CONTINENT_CODE, ecsEnabled)); + assertEquals("North America", getField(event, Field.CONTINENT_NAME, ecsEnabled)); + assertEquals("United States", getField(event, Field.COUNTRY_NAME, ecsEnabled)); + assertEquals("US", getField(event, Field.COUNTRY_CODE2, ecsEnabled)); + assertEquals("98354", getField(event, Field.POSTAL_CODE, ecsEnabled)); + assertEquals(819L, getField(event, Field.DMA_CODE, ecsEnabled)); + assertEquals("Washington", getField(event, Field.REGION_NAME, ecsEnabled)); + assertEquals("WA", getField(event, Field.REGION_CODE, ecsEnabled)); + assertEquals("US-WA", getField(event, Field.REGION_ISO_CODE, ecsEnabled)); + assertEquals("America/Los_Angeles", getField(event, Field.TIMEZONE, ecsEnabled)); + assertEquals(createLocationMap(47.2513, -122.3149), getField(event, Field.LOCATION, ecsEnabled)); + assertEquals(47.2513, getField(event, Field.LATITUDE, ecsEnabled)); + assertEquals(-122.3149, getField(event, Field.LONGITUDE, ecsEnabled)); + + if (!ecsEnabled) { + assertEquals("US", getField(event, Field.COUNTRY_CODE3, false)); + } + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void handleEventWithNoCustomFieldsShouldProperlyAddCityFields(boolean ecsEnabled) { + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOIP2_CITY, ecsEnabled, Collections.emptyList())) { + final RubyEvent rubyEvent = createRubyEvent("216.160.83.58"); + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + assertEquals(ecsEnabled, event.includes(getFieldReference(Field.REGION_ISO_CODE, ecsEnabled))); + assertEquals(!ecsEnabled, event.includes(getFieldReference(Field.REGION_CODE, ecsEnabled))); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void handleEventWithGeoIp2CountryDatabaseShouldProperlyCreateEvent(boolean ecsEnabled) { + final List supportedFields = Arrays.asList( + Field.IP, + Field.COUNTRY_CODE2, + Field.COUNTRY_NAME, + Field.CONTINENT_NAME + ); + + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOIP2_COUNTRY, ecsEnabled, supportedFields)) { + final RubyEvent rubyEvent = createRubyEvent("2a02:d5c0:0:0:0:0:0:0"); + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + assertEquals("2a02:d5c0:0:0:0:0:0:0", getField(event, Field.IP, ecsEnabled)); + assertEquals("ES", getField(event, Field.COUNTRY_CODE2, ecsEnabled)); + assertEquals("Spain", getField(event, Field.COUNTRY_NAME, ecsEnabled)); + assertEquals("Europe", getField(event, Field.CONTINENT_NAME, ecsEnabled)); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void handleEventWithGeoIp2IspDatabaseShouldProperlyCreateEvent(boolean ecsEnabled) { + final List supportedFields = Arrays.asList( + Field.IP, + Field.AUTONOMOUS_SYSTEM_NUMBER, + Field.AUTONOMOUS_SYSTEM_ORGANIZATION, + Field.ISP, + Field.ORGANIZATION + ); + + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOIP2_ISP, ecsEnabled, supportedFields)) { + final RubyEvent rubyEvent = createRubyEvent("1.128.0.1"); + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + assertEquals("1.128.0.1", getField(event, Field.IP, ecsEnabled)); + assertEquals(1221L, getField(event, Field.AUTONOMOUS_SYSTEM_NUMBER, ecsEnabled)); + assertEquals("Telstra Pty Ltd", getField(event, Field.AUTONOMOUS_SYSTEM_ORGANIZATION, ecsEnabled)); + assertEquals("Telstra Internet", getField(event, Field.ISP, ecsEnabled)); + assertEquals("Telstra Internet", getField(event, Field.ORGANIZATION, ecsEnabled)); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void handleEventWithGeoLite2AsnDatabaseShouldProperlyCreateEvent(boolean ecsEnabled) { + final List supportedFields = Arrays.asList( + Field.IP, + Field.AUTONOMOUS_SYSTEM_NUMBER, + Field.AUTONOMOUS_SYSTEM_ORGANIZATION, + Field.NETWORK + ); + + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOLITE2_ASN, ecsEnabled, supportedFields)) { + final RubyEvent rubyEvent = createRubyEvent("12.81.92.1"); + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + assertEquals("12.81.92.1", getField(event, Field.IP, ecsEnabled)); + assertEquals(7018L, getField(event, Field.AUTONOMOUS_SYSTEM_NUMBER, ecsEnabled)); + assertEquals("AT&T Services", getField(event, Field.AUTONOMOUS_SYSTEM_ORGANIZATION, ecsEnabled)); + assertEquals("12.81.92.0/22", getField(event, Field.NETWORK, ecsEnabled)); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void handleEventWithGeoIp2DomainDatabaseShouldProperlyCreateEvent(boolean ecsEnabled) { + final List supportedFields = Arrays.asList(Field.DOMAIN); + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOIP2_DOMAIN, ecsEnabled, supportedFields)) { + final RubyEvent rubyEvent = createRubyEvent("1.2.0.1"); + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + assertEquals("maxmind.com", getField(event, Field.DOMAIN, ecsEnabled)); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void handleEventWithGeoIp2EnterpriseDatabaseShouldProperlyCreateEvent(boolean ecsEnabled) { + final List supportedFields = Arrays.asList( + Field.IP, + Field.COUNTRY_CODE2, + Field.COUNTRY_NAME, + Field.CONTINENT_NAME, + Field.REGION_ISO_CODE, + Field.REGION_NAME, + Field.CITY_NAME, + Field.TIMEZONE, + Field.LOCATION, + Field.AUTONOMOUS_SYSTEM_NUMBER, + Field.AUTONOMOUS_SYSTEM_ORGANIZATION, + Field.NETWORK, + Field.HOSTING_PROVIDER, + Field.TOR_EXIT_NODE, + Field.ANONYMOUS_VPN, + Field.ANONYMOUS, + Field.PUBLIC_PROXY, + Field.RESIDENTIAL_PROXY + ); + + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOIP2_ENTERPRISE, ecsEnabled, supportedFields)) { + final RubyEvent rubyEvent = createRubyEvent("74.209.24.1"); + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + assertEquals("74.209.24.1", getField(event, Field.IP, ecsEnabled)); + assertEquals("US", getField(event, Field.COUNTRY_CODE2, ecsEnabled)); + assertEquals("United States", getField(event, Field.COUNTRY_NAME, ecsEnabled)); + assertEquals("North America", getField(event, Field.CONTINENT_NAME, ecsEnabled)); + assertEquals("US-NY", getField(event, Field.REGION_ISO_CODE, ecsEnabled)); + assertEquals("New York", getField(event, Field.REGION_NAME, ecsEnabled)); + assertEquals("Chatham", getField(event, Field.CITY_NAME, ecsEnabled)); + assertEquals("America/New_York", getField(event, Field.TIMEZONE, ecsEnabled)); + assertEquals(createLocationMap(42.3478, -73.5549), getField(event, Field.LOCATION, ecsEnabled)); + assertEquals(14671L, getField(event, Field.AUTONOMOUS_SYSTEM_NUMBER, ecsEnabled)); + assertEquals("FairPoint Communications", getField(event, Field.AUTONOMOUS_SYSTEM_ORGANIZATION, ecsEnabled)); + assertEquals("74.209.16.0/20", getField(event, Field.NETWORK, ecsEnabled)); + assertEquals(false, getField(event, Field.HOSTING_PROVIDER, ecsEnabled)); + assertEquals(false, getField(event, Field.TOR_EXIT_NODE, ecsEnabled)); + assertEquals(false, getField(event, Field.ANONYMOUS_VPN, ecsEnabled)); + assertEquals(false, getField(event, Field.ANONYMOUS, ecsEnabled)); + assertEquals(false, getField(event, Field.PUBLIC_PROXY, ecsEnabled)); + assertEquals(false, getField(event, Field.RESIDENTIAL_PROXY, ecsEnabled)); + } + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + void handleEventWithGeoIp2AnonymousIpDatabaseShouldProperlyCreateEvent(boolean ecsEnabled) { + final List supportedFields = Arrays.asList( + Field.IP, + Field.HOSTING_PROVIDER, + Field.TOR_EXIT_NODE, + Field.ANONYMOUS_VPN, + Field.ANONYMOUS, + Field.PUBLIC_PROXY, + Field.RESIDENTIAL_PROXY + ); + + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOIP2_ANONYMOUS_IP, ecsEnabled, supportedFields)) { + final RubyEvent rubyEvent = createRubyEvent("81.2.69.1"); + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + assertEquals("81.2.69.1", getField(event, Field.IP, ecsEnabled)); + assertEquals(true, getField(event, Field.HOSTING_PROVIDER, ecsEnabled)); + assertEquals(true, getField(event, Field.TOR_EXIT_NODE, ecsEnabled)); + assertEquals(true, getField(event, Field.ANONYMOUS_VPN, ecsEnabled)); + assertEquals(true, getField(event, Field.ANONYMOUS, ecsEnabled)); + assertEquals(true, getField(event, Field.PUBLIC_PROXY, ecsEnabled)); + assertEquals(true, getField(event, Field.RESIDENTIAL_PROXY, ecsEnabled)); + } + } + + @Test + void handleEventWithNoCustomFieldsShouldUseDatabasesDefaultFields() { + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOIP2_COUNTRY, true, Collections.emptyList())) { + final RubyEvent rubyEvent = createRubyEvent("216.160.83.58"); + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + for (final Field defaultField : Database.COUNTRY.getDefaultFields()) { + final String fieldReference = getFieldReference(defaultField, true); + + assertTrue(event.includes(fieldReference), () -> String.format( + "Default field %s (Fields.%s) not found on the Logstash event: %s", + fieldReference, + defaultField, + event.toMap() + )); + } + } + } + + @Test + void handleEventWithListSourceFieldShouldParseFirstIp() { + try (final GeoIPFilter filter = createFilter(MaxMindDatabases.GEOIP2_COUNTRY, true, Collections.emptyList())) { + final RubyEvent rubyEvent = createRubyEvent(new Event(Collections.singletonMap(SOURCE_FIELD, Arrays.asList("216.160.83.58", "127.0.0.1")))); + + assertTrue(filter.handleEvent(rubyEvent)); + + final Event event = rubyEvent.getEvent(); + assertEquals("216.160.83.58", getField(event, Field.IP, true)); + } + } + + private Map createLocationMap(Double lat, Double lon) { + final Map map = new HashMap<>(2); + map.put("lat", lat); + map.put("lon", lon); + return map; + } + + private Object getField(Event event, Field field, boolean ecsEnabled) { + return event.getField(getFieldReference(field, ecsEnabled)); + } + + private String getFieldReference(Field field, boolean ecsEnabled) { + final String fieldReference = (ecsEnabled ? field.getFieldReferenceECSv1() : field.getFieldReferenceLegacy()); + return String.format("[%s]%s", TARGET_FIELD, fieldReference); + } + + private RubyEvent createRubyEvent(String ipAddress) { + return createRubyEvent(new Event(Collections.singletonMap(SOURCE_FIELD, ipAddress))); + } + + private RubyEvent createRubyEvent(Event event) { + return RubyEvent.newRubyEvent(RUBY, event); + } + + private GeoIPFilter createFilter(Path databasePath, boolean ecsEnabled, List fields) { + return new GeoIPFilter( + SOURCE_FIELD, + TARGET_FIELD, + fields.stream().map(Enum::name).collect(Collectors.toList()), + databasePath.toString(), + 1000, + (ecsEnabled ? "v1" : "disabled") + ); + } +} \ No newline at end of file diff --git a/src/test/java/org/logstash/filters/geoip/MaxMindDatabases.java b/src/test/java/org/logstash/filters/geoip/MaxMindDatabases.java new file mode 100644 index 0000000..663021c --- /dev/null +++ b/src/test/java/org/logstash/filters/geoip/MaxMindDatabases.java @@ -0,0 +1,20 @@ +package org.logstash.filters.geoip; + +import java.nio.file.Path; +import java.nio.file.Paths; + +abstract class MaxMindDatabases { + private MaxMindDatabases() { /* empty */ } + + private static final Path DB_PATH = Paths.get("src/test/resources/maxmind-test-data"); + + static final Path GEOIP2_ANONYMOUS_IP = DB_PATH.resolve("GeoIP2-Anonymous-IP-Test.mmdb"); + static final Path GEOIP2_CITY = DB_PATH.resolve("GeoIP2-City-Test.mmdb"); + static final Path GEOIP2_COUNTRY = DB_PATH.resolve("GeoIP2-Country-Test.mmdb"); + static final Path GEOIP2_DOMAIN = DB_PATH.resolve("GeoIP2-Domain-Test.mmdb"); + static final Path GEOIP2_ENTERPRISE = DB_PATH.resolve("GeoIP2-Enterprise-Test.mmdb"); + static final Path GEOIP2_ISP = DB_PATH.resolve("GeoIP2-ISP-Test.mmdb"); + static final Path GEOLITE2_ASN = DB_PATH.resolve("GeoLite2-ASN-Test.mmdb"); + static final Path GEOLITE2_CITY = DB_PATH.resolve("GeoLite2-City-Test.mmdb"); + static final Path GEOLITE2_COUNTRY = DB_PATH.resolve("GeoLite2-Country-Test.mmdb"); +} diff --git a/src/test/resources/maxmind-test-data/GeoIP2-Anonymous-IP-Test.mmdb b/src/test/resources/maxmind-test-data/GeoIP2-Anonymous-IP-Test.mmdb new file mode 100644 index 0000000..17fc371 Binary files /dev/null and b/src/test/resources/maxmind-test-data/GeoIP2-Anonymous-IP-Test.mmdb differ diff --git a/src/test/resources/maxmind-test-data/GeoIP2-City-Test.mmdb b/src/test/resources/maxmind-test-data/GeoIP2-City-Test.mmdb new file mode 100644 index 0000000..e892807 Binary files /dev/null and b/src/test/resources/maxmind-test-data/GeoIP2-City-Test.mmdb differ diff --git a/src/test/resources/maxmind-test-data/GeoIP2-Country-Test.mmdb b/src/test/resources/maxmind-test-data/GeoIP2-Country-Test.mmdb new file mode 100644 index 0000000..e59fdad Binary files /dev/null and b/src/test/resources/maxmind-test-data/GeoIP2-Country-Test.mmdb differ diff --git a/src/test/resources/maxmind-test-data/GeoIP2-Domain-Test.mmdb b/src/test/resources/maxmind-test-data/GeoIP2-Domain-Test.mmdb new file mode 100644 index 0000000..d21c2a9 Binary files /dev/null and b/src/test/resources/maxmind-test-data/GeoIP2-Domain-Test.mmdb differ diff --git a/src/test/resources/maxmind-test-data/GeoIP2-Enterprise-Test.mmdb b/src/test/resources/maxmind-test-data/GeoIP2-Enterprise-Test.mmdb new file mode 100644 index 0000000..837b725 Binary files /dev/null and b/src/test/resources/maxmind-test-data/GeoIP2-Enterprise-Test.mmdb differ diff --git a/src/test/resources/maxmind-test-data/GeoIP2-ISP-Test.mmdb b/src/test/resources/maxmind-test-data/GeoIP2-ISP-Test.mmdb new file mode 100644 index 0000000..d16b0ee Binary files /dev/null and b/src/test/resources/maxmind-test-data/GeoIP2-ISP-Test.mmdb differ diff --git a/src/test/resources/maxmind-test-data/GeoLite2-ASN-Test.mmdb b/src/test/resources/maxmind-test-data/GeoLite2-ASN-Test.mmdb new file mode 100644 index 0000000..c50fbe9 Binary files /dev/null and b/src/test/resources/maxmind-test-data/GeoLite2-ASN-Test.mmdb differ diff --git a/src/test/resources/maxmind-test-data/GeoLite2-City-Test.mmdb b/src/test/resources/maxmind-test-data/GeoLite2-City-Test.mmdb new file mode 100644 index 0000000..bfbaa09 Binary files /dev/null and b/src/test/resources/maxmind-test-data/GeoLite2-City-Test.mmdb differ diff --git a/src/test/resources/maxmind-test-data/GeoLite2-Country-Test.mmdb b/src/test/resources/maxmind-test-data/GeoLite2-Country-Test.mmdb new file mode 100644 index 0000000..e714aac Binary files /dev/null and b/src/test/resources/maxmind-test-data/GeoLite2-Country-Test.mmdb differ diff --git a/src/test/resources/maxmind-test-data/README.md b/src/test/resources/maxmind-test-data/README.md new file mode 100644 index 0000000..8f870d6 --- /dev/null +++ b/src/test/resources/maxmind-test-data/README.md @@ -0,0 +1,11 @@ +## Test data for the MaxMind DB file format + +All database were copied from the official MaxMind test [repository](https://github.com/maxmind/MaxMind-DB/tree/main/test-data) +and are being used for test purpose only. + +The source-data can be found [here](https://github.com/maxmind/MaxMind-DB/tree/main/source-data). + +## Copyright and License +This software is Copyright (c) 2013 - 2024 by MaxMind, Inc. + +This is free software, licensed under the Apache License, Version 2.0 or the MIT License, at your option.