add ecs support

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 7.1.0
+  - Add ECS compatibility [#179](https://github.com/logstash-plugins/logstash-filter-geoip/pull/179)
+
 ## 7.0.1
   - [DOC] Add documentation for MaxMind database license change [#177](https://github.com/logstash-plugins/logstash-filter-geoip/pull/177)
 

docs/index.asciidoc

@@ -89,6 +89,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 [cols="<,<,<",options="header",]
 |=======================================================================
 |Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-cache_size>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-database>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-default_database_type>> |`City` or `ASN`|No
@@ -103,6 +104,20 @@ filter plugins.
 
 &nbsp;
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+* Value type is <<string,string>>
+* Supported values are:
+** `disabled`: unstructured geo data added at root level
+** `v1`: uses fields that are compatible with Elastic Common Schema (for example, `[client][geo][country_name]`)
+* Default value depends on which version of Logstash is running:
+** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
+The value of this setting affects the _default_ value of <<plugins-{type}s-{plugin}-syslog_pri_field_name>>.
+
 [id="plugins-{type}s-{plugin}-cache_size"]
 ===== `cache_size`
 
@@ -186,7 +201,11 @@ Tags the event on failure to look up geo information. This can be used in later
 ===== `target`
 
   * Value type is <<string,string>>
-  * Default value is `"geoip"`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+  ** ECS Compatibility disabled: `"geoip"`
+  ** ECS Compatibility enabled: `"client"`
+  *** `geo` field is nested in `[client][geo]`
+  *** ECS compatible values are `client`, `destination`, `host`, `observer`, `server`, `source`
 
 Specify the field into which Logstash should store the geoip data.
 This can be useful, for example, if you have `src_ip` and `dst_ip` fields and

lib/logstash/filters/geoip.rb

@@ -2,6 +2,7 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require "logstash-filter-geoip_jars"
+require "logstash/plugin_mixins/ecs_compatibility_support"
 
 
 # The GeoIP filter adds information about the geographical location of IP addresses,
@@ -31,6 +32,8 @@
 # --
 
 class LogStash::Filters::GeoIP < LogStash::Filters::Base
+  include LogStash::PluginMixins::ECSCompatibilitySupport(:disabled, :v1)
+
   config_name "geoip"
 
   # The path to the GeoLite2 database file which Logstash should use. City and ASN databases are supported.
@@ -60,14 +63,12 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
   # This can be useful, for example, if you have `src_ip` and `dst_ip` fields and
   # would like the GeoIP information of both IPs.
   #
-  # If you save the data to a target field other than `geoip` and want to use the
-  # `geo_point` related functions in Elasticsearch, you need to alter the template
-  # provided with the Elasticsearch output and configure the output to use the
-  # new template.
+  # ECS disabled default: `geoip` or ECS default: `client`
+  # ECS `geo` fields are expected to be nested at:
+  # `client`, `destination`, `host`, `observer`, `server`, `source`
   #
-  # Even if you don't use the `geo_point` mapping, the `[target][location]` field
-  # is still valid GeoJSON.
-  config :target, :validate => :string, :default => 'geoip'
+  # `geo` fields are not expected to be used directly at the root of the events
+  config :target, :validate => :string
 
   # GeoIP lookup is surprisingly expensive. This filter uses an cache to take advantage of the fact that
   # IPs agents are often found adjacent to one another in log files and rarely have a random distribution.
@@ -89,7 +90,11 @@ class LogStash::Filters::GeoIP < LogStash::Filters::Base
   config :tag_on_failure, :validate => :array, :default => ["_geoip_lookup_failure"]
 
   public
+
+  ECS_TARGET_FIELD = ['client', 'destination', 'host', 'observer', 'server', 'source'].freeze
+
   def register
+    setup_target_field
     setup_filter(select_database_path)
   end
 
@@ -108,10 +113,18 @@ def tag_unsuccessful_lookup(event)
     @tag_on_failure.each{|tag| event.tag(tag)}
   end
 
+  def setup_target_field
+    @target ||= ecs_select[disabled:'geoip', v1:'client']
+
+    if ecs_compatibility != :disabled and !ECS_TARGET_FIELD.include?(@target)
+      @logger.warn("ECS expect `target` value in #{ECS_TARGET_FIELD}")
+    end
+  end
+
   def setup_filter(database_path)
     @database = database_path
     @logger.info("Using geoip database", :path => @database)
-    @geoipfilter = org.logstash.filters.GeoIPFilter.new(@source, @target, @fields, @database, @cache_size)
+    @geoipfilter = org.logstash.filters.geoip.GeoIPFilter.new(@source, @target, @fields, @database, @cache_size, ecs_compatibility.to_s)
   end
 
   def terminate_filter

logstash-filter-geoip.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-geoip'
-  s.version         = '7.0.1'
+  s.version         = '7.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Adds geographical information about an IP address"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.1'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'insist'
   s.add_development_dependency 'benchmark-ips'

spec/filters/geoip_ecs_spec.rb

@@ -0,0 +1,95 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "insist"
+require "logstash/filters/geoip"
+require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+
+CITYDB = ::Dir.glob(::File.expand_path("../../vendor/", ::File.dirname(__FILE__))+"/GeoLite2-City.mmdb").first
+ASNDB = ::Dir.glob(::File.expand_path("../../vendor/", ::File.dirname(__FILE__))+"/GeoLite2-ASN.mmdb").first
+
+describe LogStash::Filters::GeoIP do
+  let(:options) { {} }
+  let(:plugin) { LogStash::Filters::GeoIP.new(options) }
+
+  describe "simple ip filter", :aggregate_failures do
+
+    context "when specifying the target", :ecs_compatibility_support do
+      ecs_compatibility_matrix(:disabled, :v1) do |ecs_select|
+
+        let(:ip) { "8.8.8.8" }
+        let(:event) { LogStash::Event.new("message" => ip) }
+        let(:target) { "server" }
+        let(:common_options) { {"source" => "message", "database" => CITYDB, "target" => target} }
+
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+          plugin.register
+        end
+
+        context "with city database" do
+          let(:options) { common_options }
+
+          it "should return geo in target" do
+            plugin.filter(event)
+
+            expect( event.get ecs_select[disabled: "[#{target}][ip]", v1: "[#{target}][ip]"] ).to eq ip
+            expect( event.get ecs_select[disabled: "[#{target}][country_code3]", v1: "[#{target}][country_code3]"] ).to eq 'US'
+            expect( event.get ecs_select[disabled: "[#{target}][country_code2]", v1: "[#{target}][geo][country_iso_code]"] ).to eq 'US'
+            expect( event.get ecs_select[disabled: "[#{target}][country_name]", v1: "[#{target}][geo][country_name]"] ).to eq 'United States'
+            expect( event.get ecs_select[disabled: "[#{target}][continent_code]", v1: "[#{target}][geo][continent_code]"] ).to eq 'NA'
+            expect( event.get ecs_select[disabled: "[#{target}][location][lat]", v1: "[#{target}][geo][location][lat]"] ).to eq 37.751
+            expect( event.get ecs_select[disabled: "[#{target}][location][lon]", v1: "[#{target}][geo][location][lon]"] ).to eq -97.822
+          end
+        end
+
+
+        context "with ASN database" do
+          let(:options) { common_options.merge({"database" => ASNDB}) }
+
+          it "should return geo in target" do
+            plugin.filter(event)
+
+            expect( event.get ecs_select[disabled: "[#{target}][ip]", v1: "[#{target}][ip]"] ).to eq ip
+            expect( event.get ecs_select[disabled: "[#{target}][asn]", v1: "[#{target}][as][number]"] ).to eq 15169
+            expect( event.get ecs_select[disabled: "[#{target}][as_org]", v1: "[#{target}][as][organization][name]"] ).to eq "Google LLC"
+          end
+        end
+
+        context "with customize fields" do
+          let(:fields) { ["continent_name", "timezone"] }
+          let(:options) { common_options.merge({"fields" => fields}) }
+
+          it "should return fields" do
+            plugin.filter(event)
+
+            expect( event.get ecs_select[disabled: "[#{target}][ip]", v1: "[#{target}][ip]"] ).to be_nil
+            expect( event.get ecs_select[disabled: "[#{target}][continent_name]", v1: "[#{target}][geo][continent_name]"] ).to eq "North America"
+            expect( event.get ecs_select[disabled: "[#{target}][timezone]", v1: "[#{target}][geo][timezone]"] ).to eq "America/Chicago"
+          end
+        end
+
+
+      end
+    end
+
+    context "when target is unset", :ecs_compatibility_support do
+      ecs_compatibility_matrix(:disabled, :v1) do |ecs_select|
+        let(:event) { LogStash::Event.new("message" => "8.8.8.8") }
+        let(:options) { {"source" => "message", "database" => CITYDB} }
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:ecs_compatibility).and_return(ecs_compatibility)
+          plugin.register
+        end
+
+        it "should use default target value" do
+          plugin.filter(event)
+
+          expect( event.get ecs_select[disabled: "[geoip][country_code3]", v1: "[client][country_code3]"] ).to eq 'US'
+          expect( event.get ecs_select[disabled: "[geoip][country_code2]", v1: "[client][geo][country_iso_code]"] ).to eq 'US'
+        end
+      end
+    end
+
+  end
+
+end