add ecs compatibility mode v1

Author: yaauie

Diff for: Rakefile

@@ -1 +1,28 @@
 require "logstash/devutils/rake"
+
+ECS_VERSIONS = {
+    v1: 'v1.5.0'
+}
+
+task 'vendor-ecs-schemata' do
+  download_ecs_schema(:v1, 6)
+  download_ecs_schema(:v1, 7)
+end
+
+def download_ecs_schema(ecs_major_version, es_major)
+  require 'net/http'
+  require 'json'
+  Net::HTTP.start('raw.githubusercontent.com', :use_ssl => true) do |http|
+    ecs_release_tag = ECS_VERSIONS.fetch(ecs_major_version)
+    response = http.get("/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/#{es_major}/template.json")
+    fail "#{response.code} #{response.message}" unless (200...300).cover?(response.code)
+    template_directory = File.expand_path("../lib/logstash/outputs/elasticsearch/templates/ecs-#{ecs_major_version}", __FILE__)
+    Dir.mkdir(template_directory) unless File.exists?(template_directory)
+    File.open(File.join(template_directory, "/elasticsearch-#{es_major}x.json"), "w") do |handle|
+      template = JSON.load(response.body)
+      template.fetch('index_patterns').clear
+      template.fetch('index_patterns') << 'ecs-logstash-*'
+      handle.write(JSON.pretty_generate(template))
+    end
+  end
+end

Diff for: docs/index.asciidoc

@@ -234,6 +234,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-healthcheck_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<uri,uri>>|No
@@ -393,6 +394,25 @@ If you don't set a value for this option:
 - for elasticsearch clusters 6.x: the value of 'doc' will be used;
 - for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used.
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+* Value type is <<string,string>>
+* Supported values are:
+** `disabled`: does not provide ECS-compatible templates
+** `v1`: provides defaults that are compatible with v1 of the Elastic Common Schema
+* Default value depends on which version of Logstash is running:
+** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+** Otherwise, the default value is `disabled`.
+
+Controls this plugin's desired compatibility with the https://www.elastic.co/blog/introducing-the-elastic-common-schema[Elastic Common Schema],
+including the installation of ECS-Compatible index templates.
+The value of this setting affects the _default_ values of:
+
+* <<plugins-{type}s-{plugin}-index>>
+* <<plugins-{type}s-{plugin}-template_name>>
+* <<plugins-{type}s-{plugin}-ilm_rollover_alias>>
+
 [id="plugins-{type}s-{plugin}-failure_type_logging_whitelist"]
 ===== `failure_type_logging_whitelist` 
 
@@ -500,7 +520,9 @@ NOTE: If this setting is specified, the policy must already exist in Elasticsear
 ===== `ilm_rollover_alias`
 
   * Value type is <<string,string>>
-  * Default value is `logstash`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+  ** ECS Compatibility disabled: `logstash`
+  ** ECS Compatibility enabled: `ecs-logstash` (ECS Compatibility is enabled)
 
 The rollover alias is the alias where indices managed using Index Lifecycle Management will be written to.
 
@@ -514,7 +536,9 @@ NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as `in
 ===== `index` 
 
   * Value type is <<string,string>>
-  * Default value is `"logstash-%{+yyyy.MM.dd}"`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+  ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"`
+  ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"`
 
 The index to write events to. This can be dynamic using the `%{foo}` syntax.
 The default value will partition your indices by day so you can more easily
@@ -548,7 +572,8 @@ Set the keystore password
   * Default value is `true`
 
 From Logstash 1.3 onwards, a template is applied to Elasticsearch during
-Logstash's startup if one with the name `template_name` does not already exist.
+Logstash's startup if one with the name <<plugins-{type}s-{plugin}-template_name>>
+does not already exist.
 By default, the contents of this template is the default template for
 `logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern
 `logstash-*`.  Should you require support for other index names, or would like
@@ -799,7 +824,10 @@ If not set, the included template will be used.
 ===== `template_name` 
 
   * Value type is <<string,string>>
-  * Default value is `"logstash"`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+  ** ECS Compatibility disabled: `logstash`
+  ** ECS Compatibility enabled: `ecs-logstash`
+
 
 This configuration option defines how the template is named inside Elasticsearch.
 Note that if you have used the template management features and subsequently

Diff for: lib/logstash/outputs/elasticsearch.rb

@@ -92,6 +92,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/outputs/elasticsearch/common"
   require "logstash/outputs/elasticsearch/ilm"
 
+  require 'logstash/plugin_mixins/ecs_compatibility_support'
+
   # Protocol agnostic (i.e. non-http, non-java specific) configs go here
   include(LogStash::Outputs::ElasticSearch::CommonConfigs)
 
@@ -101,6 +103,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Methods for ILM support
   include(LogStash::Outputs::ElasticSearch::Ilm)
 
+  # ecs_compatibility option, provided by Logstash core or the support adapter.
+  include(LogStash::PluginMixins::ECSCompatibilitySupport)
+
   config_name "elasticsearch"
 
   # The Elasticsearch action to perform. Valid actions are:
@@ -242,6 +247,34 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Custom Headers to send on each request to elasticsearch nodes
   config :custom_headers, :validate => :hash, :default => {}
 
+  def register
+    setup_ecs_compatibility_related_defaults
+    super
+  end
+
+  def setup_ecs_compatibility_related_defaults
+    case ecs_compatibility
+    when :disabled
+      @default_index = "logstash-%{+yyyy.MM.dd}"
+      @default_ilm_rollover_alias = "logstash"
+      @default_template_name = 'logstash'
+    when :v1
+      @default_index = "ecs-logstash-%{+yyyy.MM.dd}"
+      @default_ilm_rollover_alias = "ecs-logstash"
+      @default_template_name = 'ecs-logstash'
+    else
+      fail("unsupported ECS Compatibility `#{ecs_compatibility}`")
+    end
+
+    @index ||= default_index
+    @ilm_rollover_alias ||= default_ilm_rollover_alias
+    @template_name ||= default_template_name
+  end
+
+  attr_reader :default_index
+  attr_reader :default_ilm_rollover_alias
+  attr_reader :default_template_name
+
   # @override to handle proxy => '' as if none was set
   def config_init(params)
     proxy = params['proxy']

Diff for: lib/logstash/outputs/elasticsearch/common.rb

@@ -436,7 +436,7 @@ def safe_bulk(actions)
     end
 
     def default_index?(index)
-      @index == LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME
+      @index == @default_index
     end
 
     def dlq_enabled?

Diff for: lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -17,7 +17,7 @@ def self.included(mod)
       # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
       # LS uses Joda to format the index pattern from event timestamp.
       # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
-      mod.config :index, :validate => :string, :default => DEFAULT_INDEX_NAME
+      mod.config :index, :validate => :string
 
       mod.config :document_type, 
         :validate => :string, 
@@ -44,7 +44,7 @@ def self.included(mod)
       # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
       #
       # where `OldTemplateName` is whatever the former setting was.
-      mod.config :template_name, :validate => :string, :default => "logstash"
+      mod.config :template_name, :validate => :string
 
       # You can set the path to your own template here, if you so desire.
       # If not set, the included template will be used.
@@ -153,7 +153,7 @@ def self.included(mod)
       mod.config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => 'auto'
 
       # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index
-      mod.config :ilm_rollover_alias, :validate => :string, :default => DEFAULT_ROLLOVER_ALIAS
+      mod.config :ilm_rollover_alias, :validate => :string
 
       # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
       # {now/d} is date math, and will insert the appropriate value automatically.

Diff for: lib/logstash/outputs/elasticsearch/ilm.rb

@@ -12,7 +12,7 @@ def setup_ilm
     end
 
     def default_rollover_alias?(rollover_alias)
-      rollover_alias == LogStash::Outputs::ElasticSearch::DEFAULT_ROLLOVER_ALIAS
+      rollover_alias == default_ilm_rollover_alias
     end
 
     def ilm_alias_set?

Diff for: lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -3,14 +3,14 @@ class TemplateManager
     # To be mixed into the elasticsearch plugin base
     def self.install_template(plugin)
       return unless plugin.manage_template
-      if plugin.template.nil?
-        plugin.logger.info("Using default mapping template")
-      else
+      if plugin.template
         plugin.logger.info("Using mapping template from", :path => plugin.template)
+        template = read_template_file(plugin.template)
+      else
+        plugin.logger.info("Using default mapping template")
+        template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility)
       end
 
-
-      template = get_template(plugin.template, plugin.maximum_seen_major_version)
       add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
       plugin.logger.info("Attempting to install template", :manage_template => template)
       install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
@@ -19,9 +19,11 @@ def self.install_template(plugin)
     end
 
     private
-    def self.get_template(path, es_major_version)
-      template_path = path || default_template_path(es_major_version)
+    def self.load_default_template(es_major_version, ecs_compatibility)
+      template_path = default_template_path(es_major_version, ecs_compatibility)
       read_template_file(template_path)
+    rescue => e
+      fail "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.message}"
     end
 
     def self.install(client, template_name, template, template_overwrite)
@@ -46,9 +48,9 @@ def self.template_name(plugin)
       plugin.ilm_in_use? && !plugin.original_params.key?('template_name') ? plugin.ilm_rollover_alias : plugin.template_name
     end
 
-    def self.default_template_path(es_major_version)
+    def self.default_template_path(es_major_version, ecs_compatibility=:disabled)
       template_version = es_major_version == 1 ? 2 : es_major_version
-      default_template_name = "templates/ecs-disabled/elasticsearch-#{template_version}x.json"
+      default_template_name = "templates/ecs-#{ecs_compatibility}/elasticsearch-#{template_version}x.json"
       ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
     end