Merge pull request #220 from yaauie/ci-boundary-for-geoip-manager

ci: refactor specs with useful boundary from LS-core extension

Author: yaauie

.ci/run.sh

@@ -2,6 +2,11 @@ import:
   - logstash-plugins/.ci:travis/[email protected]
 
 env:
+  - DISTRIBUTION=default ELASTIC_STACK_VERSION=8.x
+  - DISTRIBUTION=oss ELASTIC_STACK_VERSION=8.x
+  - DISTRIBUTION=default ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
+  - DISTRIBUTION=oss ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
   - DISTRIBUTION=default ELASTIC_STACK_VERSION=7.x
+  - DISTRIBUTION=oss ELASTIC_STACK_VERSION=7.x
   - DISTRIBUTION=default ELASTIC_STACK_VERSION=7.x SNAPSHOT=true
-  - DISTRIBUTION=default ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
+  - DISTRIBUTION=oss ELASTIC_STACK_VERSION=7.x SNAPSHOT=true

.travis.yml

@@ -1,67 +1,97 @@
 # encoding: utf-8
+require 'pathname'
 require "logstash/devutils/rspec/spec_helper"
 require "insist"
 require "logstash/filters/geoip"
 require_relative 'test_helper'
 
 describe LogStash::Filters::GeoIP do
+  context "when no database_path is given" do
 
-  before(:each) do
-    ::File.delete(METADATA_PATH) if ::File.exist?(METADATA_PATH)
-  end
+    let(:last_db_path_recorder) do
+      Module.new do
+        attr_reader :last_db_path
+        def setup_filter(db_path)
+          @last_db_path = db_path
+          super
+        end
+      end
+    end
 
-  describe "config without database path in LS >= 7.14", :aggregate_failures do
-    before(:each) do
-      dir_path = Stud::Temporary.directory
-      File.open(dir_path + '/uuid', 'w') { |f| f.write(SecureRandom.uuid) }
-      allow(LogStash::SETTINGS).to receive(:get).and_call_original
-      allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.downloader.enabled").and_return(true)
-      allow(LogStash::SETTINGS).to receive(:get).with("xpack.geoip.download.endpoint").and_return(nil)
-      allow(LogStash::SETTINGS).to receive(:get).with("path.data").and_return(dir_path)
+    let(:plugin_config) { Hash["source" => "[source][ip]", "target" => "[target]"] }
+    let(:plugin) { described_class.new(plugin_config).extend(last_db_path_recorder) }
+    let(:event) { LogStash::Event.new("source" => { "ip" => "173.9.34.107" }) }
+
+    shared_examples "event enrichment" do
+      it 'enriches events' do
+        plugin.register
+        plugin.filter(event)
+
+        expect(event.get("target")).to include('ip')
+      end
     end
 
-    let(:plugin) { LogStash::Filters::GeoIP.new("source" => "[target][ip]") }
+    database_management_available = (MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)) && !LogStash::OSS
+    if database_management_available
+      context "when geoip database management is available" do
+
+        let(:mock_manager) do
+          double('LogStash::Filters::Geoip::DatabaseManager').tap do |m|
+            allow(m).to receive(:subscribe_database_path) do |db_type, explicit_path, plugin_instance|
+              explicit_path || mock_managed[db_type]
+            end
+            allow(m).to receive(:unsubscribe_database_path).with(any_args)
+          end
+        end
+
+        # The extension to this plugin that lives in Logstash core will _always_ provide a valid
+        # database path, and how it does so is not the concern of this plugin. We emulate this
+        # behaviour here by copying the vendored CC-licensed db's into a temporary path
+        let(:mock_managed) do
+          managed_path = Pathname.new(temp_data_path).join("managed", Time.now.to_i.to_s).tap(&:mkpath)
+
+          managed_city_db_path = Pathname.new(DEFAULT_CITY_DB_PATH).basename.expand_path(managed_path).to_path
+          FileUtils.cp(DEFAULT_CITY_DB_PATH, managed_city_db_path)
 
-    context "restart the plugin" do
-      let(:event) { LogStash::Event.new("target" => { "ip" => "173.9.34.107" }) }
-      let(:event2) { LogStash::Event.new("target" => { "ip" => "55.159.212.43" }) }
+          managed_asn_db_path = Pathname.new(DEFAULT_ASN_DB_PATH).basename.expand_path(managed_path).to_path
+          FileUtils.cp(DEFAULT_ASN_DB_PATH, managed_asn_db_path)
 
-      it "should use the same database" do
-        unless plugin.load_database_manager?
-          logstash_path = ENV['LOGSTASH_PATH'] || '/usr/share/logstash' # docker logstash home
-          stub_const('LogStash::Environment::LOGSTASH_HOME', logstash_path)
+          {
+            'City' => managed_city_db_path,
+            'ASN' => managed_asn_db_path,
+          }
         end
 
-        plugin.register
-        plugin.filter(event)
-        plugin.close
-        first_dirname = get_metadata_city_database_name
-        plugin.register
-        plugin.filter(event2)
-        plugin.close
-        second_dirname = get_metadata_city_database_name
+        before(:each) do
+          allow_any_instance_of(described_class).to receive(:load_database_manager?).and_return(true)
+          stub_const("LogStash::Filters::Geoip::DatabaseManager", double("DatabaseManager.Class", :instance => mock_manager))
+        end
 
-        expect(first_dirname).not_to be_nil
-        expect(first_dirname).to eq(second_dirname)
-        expect(File).to exist(get_file_path(first_dirname))
+        let(:temp_data_path) { Stud::Temporary.directory }
+        after(:each) do
+          FileUtils.rm_rf(temp_data_path) if File.exist?(temp_data_path)
+        end
+
+        it "uses a managed database" do
+          plugin.register
+          plugin.filter(event)
+          expect(plugin.last_db_path).to_not be_nil
+          expect(plugin.last_db_path).to start_with(temp_data_path)
+        end
+
+        include_examples "event enrichment"
       end
-    end
-  end if MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)
-
-  describe "config without database path in LS < 7.14" do
-    context "should run in offline mode" do
-      config <<-CONFIG
-      filter {
-        geoip {
-          source => "ip"
-        }
-      }
-      CONFIG
-
-      sample("ip" => "173.9.34.107") do
-        insist { subject.get("geoip") }.include?("ip")
-        expect(::File.exist?(METADATA_PATH)).to be_falsey
+    else
+      context "when geoip database management is not available" do
+
+        include_examples "event enrichment"
+
+        it "uses a plugin-vendored database" do
+          plugin.register
+          expect(plugin.last_db_path).to_not be_nil
+          expect(plugin.last_db_path).to include("/vendor/")
+        end
       end
     end
-  end if MAJOR < 7 || (MAJOR == 7 && MINOR < 14)
+  end
 end

spec/filters/geoip_online_spec.rb

@@ -19,13 +19,13 @@
       end
     end
 
-    describe ">= 7.14" do
+    shared_examples "with database manager" do
       it "load_database_manager? should be true" do
         expect(plugin.load_database_manager?).to be_truthy
       end
-    end if MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)
+    end
 
-    describe "<= 7.13" do
+    shared_examples "without database manager" do
       it "load_database_manager? should be false" do
         expect(plugin.load_database_manager?).to be_falsey
       end
@@ -37,6 +37,24 @@
           expect(plugin.select_database_path).to eql(DEFAULT_CITY_DB_PATH)
         end
       end
-    end if MAJOR < 7 || (MAJOR == 7 && MINOR <= 13)
+    end
+
+    if MAJOR >= 8 || (MAJOR == 7 && MINOR >= 14)
+      context "Logstash >= 7.14" do
+        if LogStash::OSS
+          context "OSS-only" do
+            include_examples "without database manager"
+          end
+        else
+          context "default distro" do
+            include_examples "with database manager"
+          end
+        end
+      end
+    else
+      describe "Logstash < 7.14" do
+        include_examples "without database manager"
+      end
+    end
   end
 end

spec/filters/geoip_spec.rb

@@ -2,28 +2,46 @@
 require "digest"
 require "csv"
 
-def get_vendor_path(filename)
-  ::File.join(::File.expand_path("../../vendor/", ::File.dirname(__FILE__)), filename)
-end
-
-def get_data_dir
-  ::File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "filters", "geoip")
-end
-
-def get_file_path(filename)
-  ::File.join(get_data_dir, filename)
-end
+# Since we use Logstash's x-pack WITHOUT the LogStash::Runner,
+# we must find it relative to logstash-core and add it to the load path.
+require 'pathname'
+logstash_core_path = Gem.loaded_specs['logstash-core']&.full_gem_path or fail("logstash-core lib not found")
+logstash_xpack_load_path = Pathname.new(logstash_core_path).join("../x-pack/lib").cleanpath.to_s
+if ENV['OSS'] == "true" || !File.exists?(logstash_xpack_load_path)
+  $stderr.puts("X-PACK is not available")
+  LogStash::OSS = true
+else
+  if !$LOAD_PATH.include?(logstash_xpack_load_path)
+    $stderr.puts("ADDING LOGSTASH X-PACK to load path: #{logstash_xpack_load_path}")
+    $LOAD_PATH.unshift(logstash_xpack_load_path)
+  end
+  LogStash::OSS = false
 
-def get_metadata_city_database_name
-  if ::File.exist?(METADATA_PATH)
-    city = ::CSV.read(METADATA_PATH, headers: false).select { |row| row[0].eql?("City") }.last
-    city[3]
+  # when running in a Logstash process that has a geoip extension available, it will
+  # be loaded before this plugin is instantiated. In tests, we need to find and load the
+  # appropriate extension ourselves.
+  extension = nil
+  extension ||= begin; require 'geoip_database_management/extension'; LogStash::const_get("GeoipDatabaseManagement::Extension"); rescue Exception; nil; end
+  extension ||= begin; require 'filters/geoip/extension'; LogStash::const_get("Filters::Geoip::Extension"); rescue Exception; nil; end
+  if extension
+    $stderr.puts("loading logstash extension for geoip: #{extension}")
+    extension.new.tap do |instance|
+      # the extensions require logstash/runner even though they don't need to,
+      # resulting in _all_ extensions being loaded into the registry, including
+      # those whose dependencies are not met by this plugin's dependency graph.
+      def instance.require(path)
+        super unless path == "logstash/runner"
+      end
+    end.additionals_settings(LogStash::SETTINGS)
   else
-    nil
+    $stderr.puts("no logstash extension for geoip is available")
   end
 end
 
-METADATA_PATH = get_file_path("metadata.csv")
+def get_vendor_path(filename)
+  ::File.join(::File.expand_path("../../vendor/", ::File.dirname(__FILE__)), filename)
+end
+
 DEFAULT_CITY_DB_PATH = get_vendor_path("GeoLite2-City.mmdb")
 DEFAULT_ASN_DB_PATH = get_vendor_path("GeoLite2-ASN.mmdb")