From 7f5785d69c314b89599f512a63dedf085f7e6352 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Thu, 9 Jul 2020 21:59:48 +0000 Subject: [PATCH 1/6] noop: group-templates in prep for ecs-compatible additions --- lib/logstash/outputs/elasticsearch/template_manager.rb | 2 +- .../ecs-disabled/elasticsearch-2x.json} | 0 .../ecs-disabled/elasticsearch-5x.json} | 0 .../ecs-disabled/elasticsearch-6x.json} | 0 .../ecs-disabled/elasticsearch-7x.json} | 0 .../ecs-disabled/elasticsearch-8x.json} | 0 spec/unit/outputs/elasticsearch/template_manager_spec.rb | 6 +++--- spec/unit/outputs/elasticsearch_spec.rb | 2 +- 8 files changed, 5 insertions(+), 5 deletions(-) rename lib/logstash/outputs/elasticsearch/{elasticsearch-template-es2x.json => templates/ecs-disabled/elasticsearch-2x.json} (100%) rename lib/logstash/outputs/elasticsearch/{elasticsearch-template-es5x.json => templates/ecs-disabled/elasticsearch-5x.json} (100%) rename lib/logstash/outputs/elasticsearch/{elasticsearch-template-es6x.json => templates/ecs-disabled/elasticsearch-6x.json} (100%) rename lib/logstash/outputs/elasticsearch/{elasticsearch-template-es7x.json => templates/ecs-disabled/elasticsearch-7x.json} (100%) rename lib/logstash/outputs/elasticsearch/{elasticsearch-template-es8x.json => templates/ecs-disabled/elasticsearch-8x.json} (100%) diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 3eff19b0e..837182e5f 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -48,7 +48,7 @@ def self.template_name(plugin) def self.default_template_path(es_major_version) template_version = es_major_version == 1 ? 2 : es_major_version - default_template_name = "elasticsearch-template-es#{template_version}x.json" + default_template_name = "templates/ecs-disabled/elasticsearch-#{template_version}x.json" ::File.expand_path(default_template_name, ::File.dirname(__FILE__)) end diff --git a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json b/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json similarity index 100% rename from lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json rename to lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-2x.json diff --git a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json b/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json similarity index 100% rename from lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json rename to lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-5x.json diff --git a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json b/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json similarity index 100% rename from lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json rename to lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-6x.json diff --git a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json b/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json similarity index 100% rename from lib/logstash/outputs/elasticsearch/elasticsearch-template-es7x.json rename to lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-7x.json diff --git a/lib/logstash/outputs/elasticsearch/elasticsearch-template-es8x.json b/lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json similarity index 100% rename from lib/logstash/outputs/elasticsearch/elasticsearch-template-es8x.json rename to lib/logstash/outputs/elasticsearch/templates/ecs-disabled/elasticsearch-8x.json diff --git a/spec/unit/outputs/elasticsearch/template_manager_spec.rb b/spec/unit/outputs/elasticsearch/template_manager_spec.rb index 7773909ce..ad0591926 100644 --- a/spec/unit/outputs/elasticsearch/template_manager_spec.rb +++ b/spec/unit/outputs/elasticsearch/template_manager_spec.rb @@ -8,17 +8,17 @@ describe ".default_template_path" do context "elasticsearch 1.x" do it "chooses the 2x template" do - expect(described_class.default_template_path(1)).to match(/elasticsearch-template-es2x.json/) + expect(described_class.default_template_path(1)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json") end end context "elasticsearch 2.x" do it "chooses the 2x template" do - expect(described_class.default_template_path(2)).to match(/elasticsearch-template-es2x.json/) + expect(described_class.default_template_path(2)).to end_with("/templates/ecs-disabled/elasticsearch-2x.json") end end context "elasticsearch 5.x" do it "chooses the 5x template" do - expect(described_class.default_template_path(5)).to match(/elasticsearch-template-es5x.json/) + expect(described_class.default_template_path(5)).to end_with("/templates/ecs-disabled/elasticsearch-5x.json") end end end diff --git a/spec/unit/outputs/elasticsearch_spec.rb b/spec/unit/outputs/elasticsearch_spec.rb index 8a1703223..453c41e2e 100644 --- a/spec/unit/outputs/elasticsearch_spec.rb +++ b/spec/unit/outputs/elasticsearch_spec.rb @@ -6,7 +6,7 @@ describe LogStash::Outputs::ElasticSearch do subject { described_class.new(options) } let(:options) { {} } - let(:maximum_seen_major_version) { rand(100) } + let(:maximum_seen_major_version) { [1,2,5,6,7,8].sample } let(:do_register) { true } From 2dacfcbc09a3d5700f34b286c2af644274a0efbc Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Mon, 13 Jul 2020 21:37:43 +0000 Subject: [PATCH 2/6] consistently use `Elasticsearch#maximum_seen_major_version` By consistently using the Output's already-public method, we can eliminate the spec's need to specify behaviour of the method internal client, which it can't reliably intercept in time. --- lib/logstash/outputs/elasticsearch/common.rb | 12 ++++++------ spec/unit/outputs/error_whitelist_spec.rb | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index 39b1f05d8..c7d11fa6f 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -60,8 +60,8 @@ def successful_connection? !!maximum_seen_major_version end - def use_event_type?(client) - client.maximum_seen_major_version < 8 + def use_event_type? + maximum_seen_major_version < 8 end # Convert the event into a 3-tuple of action, params, and event @@ -74,7 +74,7 @@ def event_action_tuple(event) routing_field_name => @routing ? event.sprintf(@routing) : nil } - params[:_type] = get_event_type(event) if use_event_type?(client) + params[:_type] = get_event_type(event) if use_event_type? if @pipeline params[:pipeline] = event.sprintf(@pipeline) @@ -347,11 +347,11 @@ def get_event_type(event) type = if @document_type event.sprintf(@document_type) else - if client.maximum_seen_major_version < 6 + if maximum_seen_major_version < 6 event.get("type") || DEFAULT_EVENT_TYPE_ES6 - elsif client.maximum_seen_major_version == 6 + elsif maximum_seen_major_version == 6 DEFAULT_EVENT_TYPE_ES6 - elsif client.maximum_seen_major_version == 7 + elsif maximum_seen_major_version == 7 DEFAULT_EVENT_TYPE_ES7 else nil diff --git a/spec/unit/outputs/error_whitelist_spec.rb b/spec/unit/outputs/error_whitelist_spec.rb index 9c8b92f0b..d32bafb41 100644 --- a/spec/unit/outputs/error_whitelist_spec.rb +++ b/spec/unit/outputs/error_whitelist_spec.rb @@ -11,10 +11,10 @@ before :each do allow(subject.logger).to receive(:warn) + allow(subject).to receive(:maximum_seen_major_version).and_return(0) subject.register - allow(subject.client).to receive(:maximum_seen_major_version).and_return(0) allow(subject.client).to receive(:get_xpack_info) allow(subject.client).to receive(:bulk).and_return( { From 739d025afcbaf48670eb02f29222193d1144f405 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Mon, 13 Jul 2020 16:10:42 +0000 Subject: [PATCH 3/6] add ecs compatibility mode v1 --- .gitignore | 1 + Rakefile | 36 ++++++++++++++ docs/index.asciidoc | 47 +++++++++++++++++-- lib/logstash/outputs/elasticsearch.rb | 33 +++++++++++++ lib/logstash/outputs/elasticsearch/common.rb | 2 +- .../outputs/elasticsearch/common_configs.rb | 6 +-- lib/logstash/outputs/elasticsearch/ilm.rb | 2 +- .../outputs/elasticsearch/template_manager.rb | 21 +++++---- logstash-output-elasticsearch.gemspec | 1 + spec/integration/outputs/ilm_spec.rb | 4 +- .../elasticsearch/template_manager_spec.rb | 6 +++ 11 files changed, 139 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 2f71909a4..bb81a8862 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ target/ vendor/ /spec/fixtures/server.key /spec/fixtures/server.crt +/lib/logstash/outputs/elasticsearch/templates/ecs-v* diff --git a/Rakefile b/Rakefile index d50e796f1..d67708cde 100644 --- a/Rakefile +++ b/Rakefile @@ -1 +1,37 @@ require "logstash/devutils/rake" + +ECS_VERSIONS = { + v1: 'v1.5.0' +} + +ECS_LOGSTASH_INDEX_PATTERNS = %w( + ecs-logstash-* +) + +task :'vendor-ecs-schemata' do + download_ecs_schema(:v1, 6) + download_ecs_schema(:v1, 7) +end +task :vendor => :'vendor-ecs-schemata' + +def download_ecs_schema(ecs_major_version, es_major) + $stderr.puts("Vendoring ECS #{ecs_major_version} template for Elasticsearch #{es_major}") + require 'net/http' + require 'json' + Net::HTTP.start('raw.githubusercontent.com', :use_ssl => true) do |http| + ecs_release_tag = ECS_VERSIONS.fetch(ecs_major_version) + response = http.get("/elastic/ecs/#{ecs_release_tag}/generated/elasticsearch/#{es_major}/template.json") + fail "#{response.code} #{response.message}" unless (200...300).cover?(response.code.to_i) + template_directory = File.expand_path("../lib/logstash/outputs/elasticsearch/templates/ecs-#{ecs_major_version}", __FILE__) + Dir.mkdir(template_directory) unless File.exists?(template_directory) + File.open(File.join(template_directory, "/elasticsearch-#{es_major}x.json"), "w") do |handle| + handle.write(replace_index_patterns(response.body, ECS_LOGSTASH_INDEX_PATTERNS)) + end + end +end + +def replace_index_patterns(template_json, replacement_index_patterns) + template_obj = JSON.load(template_json) + template_obj.update('index_patterns' => replacement_index_patterns) + JSON.pretty_generate(template_obj) +end diff --git a/docs/index.asciidoc b/docs/index.asciidoc index a8743aab8..e6ad53740 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -58,6 +58,17 @@ https://www.elastic.co/cloud/elasticsearch-service[hosted {es} Service] on Elastic Cloud. The Elasticsearch Service is available on AWS, Google Cloud Platform, and Microsoft Azure. {ess-trial}[Try the {es} Service for free]. +==== Compatibility with the Elastic Common Schema (ECS) + +This plugin will persist events to Elasticsearch in the shape produced by +your pipeline, and _cannot_ be used to re-shape the event structure into a +shape that complies with ECS. + +However, the Elasticsearch Index Templates it manages can be configured to +be ECS-compatible by setting <>. +By having an ECS-compatible template in-place, we can ensure that Elasticsearch +is prepared to create and index fields in a way that is compatible with ECS. + ==== Writing to different indices: best practices [NOTE] @@ -234,6 +245,7 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |<>|No | <> |<>|No +| <> | <>|No | <> |<>|No | <> |<>|No | <> |<>|No @@ -393,6 +405,25 @@ If you don't set a value for this option: - for elasticsearch clusters 6.x: the value of 'doc' will be used; - for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. +[id="plugins-{type}s-{plugin}-ecs_compatibility"] +===== `ecs_compatibility` + +* Value type is <> +* Supported values are: +** `disabled`: does not provide ECS-compatible templates +** `v1`: provides defaults that are compatible with v1 of the Elastic Common Schema +* Default value depends on which version of Logstash is running: +** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default +** Otherwise, the default value is `disabled`. + +Controls this plugin's desired compatibility with the https://www.elastic.co/blog/introducing-the-elastic-common-schema[Elastic Common Schema], +including the installation of ECS-Compatible index templates. +The value of this setting affects the _default_ values of: + +* <> +* <> +* <> + [id="plugins-{type}s-{plugin}-failure_type_logging_whitelist"] ===== `failure_type_logging_whitelist` @@ -500,7 +531,9 @@ NOTE: If this setting is specified, the policy must already exist in Elasticsear ===== `ilm_rollover_alias` * Value type is <> - * Default value is `logstash` + * Default value depends on whether <> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` (ECS Compatibility is enabled) The rollover alias is the alias where indices managed using Index Lifecycle Management will be written to. @@ -514,7 +547,9 @@ NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as `in ===== `index` * Value type is <> - * Default value is `"logstash-%{+yyyy.MM.dd}"` + * Default value depends on whether <> is enabled: + ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"` + ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"` The index to write events to. This can be dynamic using the `%{foo}` syntax. The default value will partition your indices by day so you can more easily @@ -548,7 +583,8 @@ Set the keystore password * Default value is `true` From Logstash 1.3 onwards, a template is applied to Elasticsearch during -Logstash's startup if one with the name `template_name` does not already exist. +Logstash's startup if one with the name <> +does not already exist. By default, the contents of this template is the default template for `logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern `logstash-*`. Should you require support for other index names, or would like @@ -799,7 +835,10 @@ If not set, the included template will be used. ===== `template_name` * Value type is <> - * Default value is `"logstash"` + * Default value depends on whether <> is enabled: + ** ECS Compatibility disabled: `logstash` + ** ECS Compatibility enabled: `ecs-logstash` + This configuration option defines how the template is named inside Elasticsearch. Note that if you have used the template management features and subsequently diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index 5639bad3a..c9cdd0465 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -92,6 +92,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base require "logstash/outputs/elasticsearch/common" require "logstash/outputs/elasticsearch/ilm" + require 'logstash/plugin_mixins/ecs_compatibility_support' + # Protocol agnostic (i.e. non-http, non-java specific) configs go here include(LogStash::Outputs::ElasticSearch::CommonConfigs) @@ -101,6 +103,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base # Methods for ILM support include(LogStash::Outputs::ElasticSearch::Ilm) + # ecs_compatibility option, provided by Logstash core or the support adapter. + include(LogStash::PluginMixins::ECSCompatibilitySupport) + config_name "elasticsearch" # The Elasticsearch action to perform. Valid actions are: @@ -242,6 +247,34 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base # Custom Headers to send on each request to elasticsearch nodes config :custom_headers, :validate => :hash, :default => {} + def initialize(*params) + super + setup_ecs_compatibility_related_defaults + end + + def setup_ecs_compatibility_related_defaults + case ecs_compatibility + when :disabled + @default_index = "logstash-%{+yyyy.MM.dd}" + @default_ilm_rollover_alias = "logstash" + @default_template_name = 'logstash' + when :v1 + @default_index = "ecs-logstash-%{+yyyy.MM.dd}" + @default_ilm_rollover_alias = "ecs-logstash" + @default_template_name = 'ecs-logstash' + else + fail("unsupported ECS Compatibility `#{ecs_compatibility}`") + end + + @index ||= default_index + @ilm_rollover_alias ||= default_ilm_rollover_alias + @template_name ||= default_template_name + end + + attr_reader :default_index + attr_reader :default_ilm_rollover_alias + attr_reader :default_template_name + # @override to handle proxy => '' as if none was set def config_init(params) proxy = params['proxy'] diff --git a/lib/logstash/outputs/elasticsearch/common.rb b/lib/logstash/outputs/elasticsearch/common.rb index c7d11fa6f..61550c728 100644 --- a/lib/logstash/outputs/elasticsearch/common.rb +++ b/lib/logstash/outputs/elasticsearch/common.rb @@ -436,7 +436,7 @@ def safe_bulk(actions) end def default_index?(index) - @index == LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME + @index == @default_index end def dlq_enabled? diff --git a/lib/logstash/outputs/elasticsearch/common_configs.rb b/lib/logstash/outputs/elasticsearch/common_configs.rb index b686f774c..dc2efea56 100644 --- a/lib/logstash/outputs/elasticsearch/common_configs.rb +++ b/lib/logstash/outputs/elasticsearch/common_configs.rb @@ -17,7 +17,7 @@ def self.included(mod) # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}. # LS uses Joda to format the index pattern from event timestamp. # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here]. - mod.config :index, :validate => :string, :default => DEFAULT_INDEX_NAME + mod.config :index, :validate => :string mod.config :document_type, :validate => :string, @@ -44,7 +44,7 @@ def self.included(mod) # `curl -XDELETE ` # # where `OldTemplateName` is whatever the former setting was. - mod.config :template_name, :validate => :string, :default => "logstash" + mod.config :template_name, :validate => :string # You can set the path to your own template here, if you so desire. # If not set, the included template will be used. @@ -153,7 +153,7 @@ def self.included(mod) mod.config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => 'auto' # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index - mod.config :ilm_rollover_alias, :validate => :string, :default => DEFAULT_ROLLOVER_ALIAS + mod.config :ilm_rollover_alias, :validate => :string # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002” # {now/d} is date math, and will insert the appropriate value automatically. diff --git a/lib/logstash/outputs/elasticsearch/ilm.rb b/lib/logstash/outputs/elasticsearch/ilm.rb index a9f43f12a..1a891d5af 100644 --- a/lib/logstash/outputs/elasticsearch/ilm.rb +++ b/lib/logstash/outputs/elasticsearch/ilm.rb @@ -12,7 +12,7 @@ def setup_ilm end def default_rollover_alias?(rollover_alias) - rollover_alias == LogStash::Outputs::ElasticSearch::DEFAULT_ROLLOVER_ALIAS + rollover_alias == default_ilm_rollover_alias end def ilm_alias_set? diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 837182e5f..20f2cbd2c 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -3,14 +3,15 @@ class TemplateManager # To be mixed into the elasticsearch plugin base def self.install_template(plugin) return unless plugin.manage_template - if plugin.template.nil? - plugin.logger.info("Using default mapping template") - else + if plugin.template plugin.logger.info("Using mapping template from", :path => plugin.template) + template = read_template_file(plugin.template) + else + plugin.logger.info("Using a default mapping template", :es_version => plugin.maximum_seen_major_version, + :ecs_compatibility => plugin.ecs_compatibility) + template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility) end - - template = get_template(plugin.template, plugin.maximum_seen_major_version) add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use? plugin.logger.info("Attempting to install template", :manage_template => template) install(plugin.client, template_name(plugin), template, plugin.template_overwrite) @@ -19,9 +20,11 @@ def self.install_template(plugin) end private - def self.get_template(path, es_major_version) - template_path = path || default_template_path(es_major_version) + def self.load_default_template(es_major_version, ecs_compatibility) + template_path = default_template_path(es_major_version, ecs_compatibility) read_template_file(template_path) + rescue => e + fail "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}" end def self.install(client, template_name, template, template_overwrite) @@ -46,9 +49,9 @@ def self.template_name(plugin) plugin.ilm_in_use? && !plugin.original_params.key?('template_name') ? plugin.ilm_rollover_alias : plugin.template_name end - def self.default_template_path(es_major_version) + def self.default_template_path(es_major_version, ecs_compatibility=:disabled) template_version = es_major_version == 1 ? 2 : es_major_version - default_template_name = "templates/ecs-disabled/elasticsearch-#{template_version}x.json" + default_template_name = "templates/ecs-#{ecs_compatibility}/elasticsearch-#{template_version}x.json" ::File.expand_path(default_template_name, ::File.dirname(__FILE__)) end diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index 819c44b50..ecf00b11c 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -25,6 +25,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0'] s.add_runtime_dependency 'cabin', ['~> 0.6'] s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" + s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0' s.add_development_dependency 'logstash-codec-plain' s.add_development_dependency 'logstash-devutils' diff --git a/spec/integration/outputs/ilm_spec.rb b/spec/integration/outputs/ilm_spec.rb index 142e0cdd6..9ec211d94 100644 --- a/spec/integration/outputs/ilm_spec.rb +++ b/spec/integration/outputs/ilm_spec.rb @@ -249,9 +249,9 @@ } let (:small_max_doc_policy) { max_docs_policy(3) } let (:large_max_doc_policy) { max_docs_policy(1000000) } - let (:expected_index) { LogStash::Outputs::ElasticSearch::DEFAULT_ROLLOVER_ALIAS } + let (:expected_index) { elasticsearch_output_plugin.default_ilm_rollover_alias } - subject { LogStash::Outputs::ElasticSearch.new(settings) } + subject(:elasticsearch_output_plugin) { LogStash::Outputs::ElasticSearch.new(settings) } before :each do # Delete all templates first. diff --git a/spec/unit/outputs/elasticsearch/template_manager_spec.rb b/spec/unit/outputs/elasticsearch/template_manager_spec.rb index ad0591926..af5e43863 100644 --- a/spec/unit/outputs/elasticsearch/template_manager_spec.rb +++ b/spec/unit/outputs/elasticsearch/template_manager_spec.rb @@ -22,4 +22,10 @@ end end end + + context 'when ECS v1 is requested' do + it 'resolves' do + expect(described_class.default_template_path(7, :v1)).to end_with("/templates/ecs-v1/elasticsearch-7x.json") + end + end end From cd869892f9e2b1f75c5444a7a0fb38a288435140 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Tue, 14 Jul 2020 18:34:51 +0000 Subject: [PATCH 4/6] bump to 10.6.0 --- CHANGELOG.md | 3 +++ logstash-output-elasticsearch.gemspec | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed27c8476..351939672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 10.6.0 + - Added `ecs_compatiblity` mode, for managing ECS-compatable templates [#952](https://github.com/logstash-plugins/logstash-output-elasticsearch/issue/952) + ## 10.5.1 - [DOC] Removed outdated compatibility notices, reworked cloud notice, and fixed formatting for `hosts` examples [#938](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/938) diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index ecf00b11c..3f7bb7b72 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-elasticsearch' - s.version = '10.5.1' + s.version = '10.6.0' s.licenses = ['apache-2.0'] s.summary = "Stores logs in Elasticsearch" From 03d661758fe02952fd3c2be60846edd7ee223c6f Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Tue, 14 Jul 2020 21:45:39 +0000 Subject: [PATCH 5/6] docs: ECS touchups --- docs/index.asciidoc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index e6ad53740..af2b092d3 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -62,12 +62,15 @@ Platform, and Microsoft Azure. {ess-trial}[Try the {es} Service for free]. This plugin will persist events to Elasticsearch in the shape produced by your pipeline, and _cannot_ be used to re-shape the event structure into a -shape that complies with ECS. +shape that complies with ECS. To produce events that fully comply with ECS, +you will need to populate ECS-defined fields throughout your pipeline +definition. However, the Elasticsearch Index Templates it manages can be configured to be ECS-compatible by setting <>. By having an ECS-compatible template in-place, we can ensure that Elasticsearch -is prepared to create and index fields in a way that is compatible with ECS. +is prepared to create and index fields in a way that is compatible with ECS, +and will correctly reject events with fields that conflict and cannot be coerced. ==== Writing to different indices: best practices @@ -416,8 +419,8 @@ If you don't set a value for this option: ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default ** Otherwise, the default value is `disabled`. -Controls this plugin's desired compatibility with the https://www.elastic.co/blog/introducing-the-elastic-common-schema[Elastic Common Schema], -including the installation of ECS-Compatible index templates. +Controls this plugin's compatibility with the {ecs-ref}}[Elastic Common Schema (ECS)], +including the installation of ECS-compatible index templates. The value of this setting affects the _default_ values of: * <> @@ -533,7 +536,7 @@ NOTE: If this setting is specified, the policy must already exist in Elasticsear * Value type is <> * Default value depends on whether <> is enabled: ** ECS Compatibility disabled: `logstash` - ** ECS Compatibility enabled: `ecs-logstash` (ECS Compatibility is enabled) + ** ECS Compatibility enabled: `ecs-logstash` The rollover alias is the alias where indices managed using Index Lifecycle Management will be written to. From 9dd0b562c793782925bb10a912adc5b919f17214 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Tue, 14 Jul 2020 14:48:39 -0700 Subject: [PATCH 6/6] docs: ECS touchup Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/index.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index af2b092d3..74590ba1c 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -68,7 +68,7 @@ definition. However, the Elasticsearch Index Templates it manages can be configured to be ECS-compatible by setting <>. -By having an ECS-compatible template in-place, we can ensure that Elasticsearch +By having an ECS-compatible template in place, we can ensure that Elasticsearch is prepared to create and index fields in a way that is compatible with ECS, and will correctly reject events with fields that conflict and cannot be coerced.