Add 'metadata' option to include @metadata field in output

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.1.0
+ - feature: Add 'metadata' option that enables inclusion of the @metadata field in the output.
+
 ## 3.0.6
   - Support flush method, see https://github.com/logstash-plugins/logstash-codec-json_lines/pull/35
 

docs/index.asciidoc

@@ -36,6 +36,7 @@ Therefore this codec cannot work with line oriented inputs.
 |Setting |Input type|Required
 | <<plugins-{type}s-{plugin}-charset>> |<<string,string>>, one of `["ASCII-8BIT", "UTF-8", "US-ASCII", "Big5", "Big5-HKSCS", "Big5-UAO", "CP949", "Emacs-Mule", "EUC-JP", "EUC-KR", "EUC-TW", "GB2312", "GB18030", "GBK", "ISO-8859-1", "ISO-8859-2", "ISO-8859-3", "ISO-8859-4", "ISO-8859-5", "ISO-8859-6", "ISO-8859-7", "ISO-8859-8", "ISO-8859-9", "ISO-8859-10", "ISO-8859-11", "ISO-8859-13", "ISO-8859-14", "ISO-8859-15", "ISO-8859-16", "KOI8-R", "KOI8-U", "Shift_JIS", "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "Windows-31J", "Windows-1250", "Windows-1251", "Windows-1252", "IBM437", "IBM737", "IBM775", "CP850", "IBM852", "CP852", "IBM855", "CP855", "IBM857", "IBM860", "IBM861", "IBM862", "IBM863", "IBM864", "IBM865", "IBM866", "IBM869", "Windows-1258", "GB1988", "macCentEuro", "macCroatian", "macCyrillic", "macGreek", "macIceland", "macRoman", "macRomania", "macThai", "macTurkish", "macUkraine", "CP950", "CP951", "IBM037", "stateless-ISO-2022-JP", "eucJP-ms", "CP51932", "EUC-JIS-2004", "GB12345", "ISO-2022-JP", "ISO-2022-JP-2", "CP50220", "CP50221", "Windows-1256", "Windows-1253", "Windows-1255", "Windows-1254", "TIS-620", "Windows-874", "Windows-1257", "MacJapanese", "UTF-7", "UTF8-MAC", "UTF-16", "UTF-32", "UTF8-DoCoMo", "SJIS-DoCoMo", "UTF8-KDDI", "SJIS-KDDI", "ISO-2022-JP-KDDI", "stateless-ISO-2022-JP-KDDI", "UTF8-SoftBank", "SJIS-SoftBank", "BINARY", "CP437", "CP737", "CP775", "IBM850", "CP857", "CP860", "CP861", "CP862", "CP863", "CP864", "CP865", "CP866", "CP869", "CP1258", "Big5-HKSCS:2008", "ebcdic-cp-us", "eucJP", "euc-jp-ms", "EUC-JISX0213", "eucKR", "eucTW", "EUC-CN", "eucCN", "CP936", "ISO2022-JP", "ISO2022-JP2", "ISO8859-1", "ISO8859-2", "ISO8859-3", "ISO8859-4", "ISO8859-5", "ISO8859-6", "CP1256", "ISO8859-7", "CP1253", "ISO8859-8", "CP1255", "ISO8859-9", "CP1254", "ISO8859-10", "ISO8859-11", "CP874", "ISO8859-13", "CP1257", "ISO8859-14", "ISO8859-15", "ISO8859-16", "CP878", "MacJapan", "ASCII", "ANSI_X3.4-1968", "646", "CP65000", "CP65001", "UTF-8-MAC", "UTF-8-HFS", "UCS-2BE", "UCS-4BE", "UCS-4LE", "CP932", "csWindows31J", "SJIS", "PCK", "CP1250", "CP1251", "CP1252", "external", "locale"]`|No
 | <<plugins-{type}s-{plugin}-delimiter>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-metadata>> |<<boolean,boolean>>|No
 |=======================================================================
 
 &nbsp;
@@ -64,4 +65,12 @@ For nxlog users, you'll want to set this to `CP1252`
 
 Change the delimiter that separates lines
 
+[id="plugins-{type}s-{plugin}-metadata"]
+===== `metadata`
 
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+  * This option is deprecated and may be removed in a future major release of the plugin.
+
+If true, the event's metadata (the `@metadata` field and any
+subfields) will be included when used as an output codec.

lib/logstash/codecs/json_lines.rb

@@ -28,12 +28,22 @@ class LogStash::Codecs::JSONLines < LogStash::Codecs::Base
   # Change the delimiter that separates lines
   config :delimiter, :validate => :string, :default => "\n"
 
+  # If true, the event's metadata (the `@metadata` field) will be
+  # included when used as an output codec.
+  config :metadata, :validate => :boolean, :default => false,
+    :deprecated => "Option may be removed in a future release."
+
   public
 
   def register
     @buffer = FileWatch::BufferedTokenizer.new(@delimiter)
     @converter = LogStash::Util::Charset.new(@charset)
     @converter.logger = @logger
+    if @metadata
+      @encoder = method(:encode_with_metadata)
+    else
+      @encoder = method(:encode_default)
+    end
   end
 
   def decode(data, &block)
@@ -43,9 +53,7 @@ def decode(data, &block)
   end
 
   def encode(event)
-    # Tack on a @delimiter for now because previously most of logstash's JSON
-    # outputs emitted one per line, and whitespace is OK in json.
-    @on_event.call(event, "#{event.to_json}#{@delimiter}")
+    @encoder.call(event)
   end
 
   def flush(&block)
@@ -57,6 +65,18 @@ def flush(&block)
 
   private
 
+  def encode_default(event)
+    # Tack on a @delimiter for now because previously most of logstash's JSON
+    # outputs emitted one per line, and whitespace is OK in json.
+    @on_event.call(event, "#{event.to_json}#{@delimiter}")
+  end
+
+  def encode_with_metadata(event)
+    # Tack on a @delimiter for now because previously most of logstash's JSON
+    # outputs emitted one per line, and whitespace is OK in json.
+    @on_event.call(event, "#{LogStash::Json.dump(event.to_hash_with_metadata)}#{@delimiter}")
+  end
+
   # from_json_parse uses the Event#from_json method to deserialize and directly produce events
   def from_json_parse(json, &block)
     LogStash::Event.from_json(json).each { |event| yield event }

logstash-codec-json_lines.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-codec-json_lines'
-  s.version         = '3.0.6'
+  s.version         = '3.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Reads and writes newline-delimited JSON"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/codecs/json_lines_spec.rb

@@ -120,13 +120,13 @@
   end
 
   context "#encode" do
-    let(:data) { { LogStash::Event::TIMESTAMP => "2015-12-07T11:37:00.000Z", "foo" => "bar", "baz" => {"bah" => ["a","b","c"]}} }
-    let(:event) { LogStash::Event.new(data) }
+    let(:data) { { LogStash::Event::TIMESTAMP => "2015-12-07T11:37:00.000Z", "foo" => "bar", "baz" => {"bah" => ["a","b","c"]}, "@metadata" => {"metafield" => "metavalue"} } }
+    let(:event) { LogStash::Event.new(data.clone) }
 
     it "should return json data" do
       got_event = false
       subject.on_event do |e, d|
-        insist { d } == "#{LogStash::Event.new(data).to_json}\n"
+        insist { d } == "#{LogStash::Event.new(data.clone).to_json}\n"
         insist { LogStash::Json.load(d)["foo"] } == data["foo"]
         insist { LogStash::Json.load(d)["baz"] } == data["baz"]
         insist { LogStash::Json.load(d)["bah"] } == data["bah"]
@@ -136,13 +136,35 @@
       insist { got_event }
     end
 
+    it "should not include metadata in the output" do
+      got_event = false
+      subject.on_event do |e, d|
+        insist { LogStash::Json.load(d).has_key? "@metadata" } == false
+        got_event = true
+      end
+      subject.encode(event)
+      insist { got_event }
+    end
+
     context "when using custom delimiter" do
       let(:delimiter) { "|" }
       let(:codec_options) { { "delimiter" => delimiter } }
 
       it "should decode multiple lines separated by the delimiter" do
         subject.on_event do |e, d|
-          insist { d } == "#{LogStash::Event.new(data).to_json}#{delimiter}"
+          insist { d } == "#{LogStash::Event.new(data.clone).to_json}#{delimiter}"
+        end
+        subject.encode(event)
+      end
+    end
+
+    context "when enabling metadata output" do
+      let(:codec_options) { { "metadata" => true } }
+
+      it "should include the @metadata field" do
+        subject.on_event do |e, d|
+          insist { d } == "#{LogStash::Json.dump(LogStash::Event.new(data.clone).to_hash_with_metadata)}\n"
+          insist { LogStash::Json.load(d)["@metadata"] } == data["@metadata"]
         end
         subject.encode(event)
       end