From af877b6efa09c8f4cc02ba32a0a8cb3201e5b83c Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Wed, 27 Jul 2016 15:43:44 -0600 Subject: [PATCH 1/3] Add path.data functionality But maintain reverse compatibility for older versions of Logstash I don't yet know how to test this... --- CHANGELOG.md | 3 ++- CONTRIBUTORS | 1 + lib/logstash/inputs/file.rb | 11 +++++++++++ logstash-input-file.gemspec | 3 +-- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 692d0cf..06ea1e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ +## 3.1.0 + - Use native `--path.data` for Logstash 5.0 for sincedb files. ## 3.0.3 - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 - ## 3.0.2 - relax constrains of `logstash-devutils` see https://github.com/elastic/logstash-devutils/issues/48 ## 3.0.1 diff --git a/CONTRIBUTORS b/CONTRIBUTORS index ea05590..d89064c 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -18,6 +18,7 @@ Contributors: * elliot moore (em295) * yjpa7145 * Guy Boertje (guyboertje) +* Aaron Mildenstein (untergeek) Note: If you've sent us patches, bug reports, or otherwise contributed to Logstash, and you aren't on the list above and want to be, please let us know diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 04dc36c..c58534c 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -5,6 +5,7 @@ require "pathname" require "socket" # for Socket.gethostname +require "fileutils" # Stream events from files, normally by tailing them in a manner # similar to `tail -0F` but optionally reading them from the @@ -170,6 +171,7 @@ def register require "digest/md5" @logger.info("Registering file input", :path => @path) @host = Socket.gethostname.force_encoding(Encoding::UTF_8) + @settings = defined?(LogStash::SETTINGS) ? LogStash::SETTINGS : nil @tail_config = { :exclude => @exclude, @@ -189,6 +191,15 @@ def register end if @sincedb_path.nil? + if @settings + datapath = File.join(@settings.get_value("path.data"), "plugins", "inputs", "file") + # Ensure that the filepath exists before writing, since it's deeply nested. + FileUtils::mkdir_p datapath + @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) + + # This section is going to be deprecated eventually, as path.data will be + # the default, not an environment variable (SINCEDB_DIR or HOME) + if @sincedb_path.nil? # If it is _still_ nil... if ENV["SINCEDB_DIR"].nil? && ENV["HOME"].nil? @logger.error("No SINCEDB_DIR or HOME environment variable set, I don't know where " \ "to keep track of the files I'm watching. Either set " \ diff --git a/logstash-input-file.gemspec b/logstash-input-file.gemspec index 07fa3c4..781273c 100644 --- a/logstash-input-file.gemspec +++ b/logstash-input-file.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-file' - s.version = '3.0.3' + s.version = '3.1.0' s.licenses = ['Apache License (2.0)'] s.summary = "Stream events from files." s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" @@ -32,4 +32,3 @@ Gem::Specification.new do |s| s.add_development_dependency 'logstash-codec-json' s.add_development_dependency 'rspec-sequencing' end - From c81ca49b331c5c7a6e6f62bcfa7c0819185fe4f1 Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Thu, 28 Jul 2016 11:36:45 -0600 Subject: [PATCH 2/3] End if statements I really need to stop thinking in Python... --- lib/logstash/inputs/file.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index c58534c..39f838f 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -196,6 +196,8 @@ def register # Ensure that the filepath exists before writing, since it's deeply nested. FileUtils::mkdir_p datapath @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path.join(","))) + end + end # This section is going to be deprecated eventually, as path.data will be # the default, not an environment variable (SINCEDB_DIR or HOME) From a6ccd12df5fc5bb864a5ac5c8981e147e8bae399 Mon Sep 17 00:00:00 2001 From: Aaron Mildenstein Date: Thu, 28 Jul 2016 12:01:38 -0600 Subject: [PATCH 3/3] Change settings to not be an ivar And document what settings is for, and why. --- lib/logstash/inputs/file.rb | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 39f838f..cc2324f 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -171,7 +171,9 @@ def register require "digest/md5" @logger.info("Registering file input", :path => @path) @host = Socket.gethostname.force_encoding(Encoding::UTF_8) - @settings = defined?(LogStash::SETTINGS) ? LogStash::SETTINGS : nil + # This check is Logstash 5 specific. If the class does not exist, and it + # won't in older versions of Logstash, then we need to set it to nil. + settings = defined?(LogStash::SETTINGS) ? LogStash::SETTINGS : nil @tail_config = { :exclude => @exclude, @@ -191,8 +193,8 @@ def register end if @sincedb_path.nil? - if @settings - datapath = File.join(@settings.get_value("path.data"), "plugins", "inputs", "file") + if settings + datapath = File.join(settings.get_value("path.data"), "plugins", "inputs", "file") # Ensure that the filepath exists before writing, since it's deeply nested. FileUtils::mkdir_p datapath @sincedb_path = File.join(datapath, ".sincedb_" + Digest::MD5.hexdigest(@path.join(",")))