From d84bb0aab4073e204e87a6320afd79753a7faa7d Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Wed, 23 May 2018 19:47:48 +0100 Subject: [PATCH 1/2] add string based time duration support. --- lib/logstash/inputs/file.rb | 25 ++++++-- lib/logstash/inputs/friendly_durations.rb | 46 +++++++++++++++ spec/inputs/friendly_durations_spec.rb | 70 +++++++++++++++++++++++ 3 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 lib/logstash/inputs/friendly_durations.rb create mode 100644 spec/inputs/friendly_durations_spec.rb diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index d7e068e..39b9b1f 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -11,6 +11,7 @@ require_relative "file_listener" require_relative "delete_completed_file_handler" require_relative "log_completed_file_handler" +require_relative "friendly_durations" require "filewatch/bootstrap" # Stream events from files, normally by tailing them in a manner @@ -109,7 +110,7 @@ class File < LogStash::Inputs::Base # How often (in seconds) we stat files to see if they have been modified. # Increasing this interval will decrease the number of system calls we make, # but increase the time to detect new log lines. - config :stat_interval, :validate => :number, :default => 1 + config :stat_interval, :validate => [FriendlyDurations, "seconds"], :default => 1 # How often (in seconds) we expand the filename patterns in the # `path` option to discover new files to watch. @@ -123,7 +124,7 @@ class File < LogStash::Inputs::Base # How often (in seconds) to write a since database with the current position of # monitored log files. - config :sincedb_write_interval, :validate => :number, :default => 15 + config :sincedb_write_interval, :validate => [FriendlyDurations, "seconds"], :default => 15 # Choose where Logstash starts initially reading files: at the beginning or # at the end. The default behavior treats files like live streams and thus @@ -145,7 +146,7 @@ class File < LogStash::Inputs::Base # After its discovery, if an ignored file is modified it is no # longer ignored and any new data is read. By default, this option is # disabled. Note this unit is in seconds. - config :ignore_older, :validate => :number + config :ignore_older, :validate => [FriendlyDurations, "seconds"] # The file input closes any files that were last read the specified # timespan in seconds ago. @@ -154,7 +155,7 @@ class File < LogStash::Inputs::Base # reopening when new data is detected. If reading, the file will be closed # after closed_older seconds from when the last bytes were read. # The default is 1 hour - config :close_older, :validate => :number, :default => 1 * 60 * 60 + config :close_older, :validate => [FriendlyDurations, "seconds"], :default => "1 hour" # What is the maximum number of file_handles that this input consumes # at any one time. Use close_older to close some files if you need to @@ -191,7 +192,7 @@ class File < LogStash::Inputs::Base # If no changes are detected in tracked files in the last N days their sincedb # tracking record will expire and not be persisted. # This option protects against the well known inode recycling problem. (add reference) - config :sincedb_clean_after, :validate => :number, :default => 14 # days + config :sincedb_clean_after, :validate => [FriendlyDurations, "days"], :default => "14 days" # days # File content is read off disk in blocks or chunks, then using whatever the set delimiter # is, lines are extracted from the chunk. Specify the size in bytes of each chunk. @@ -222,6 +223,20 @@ class File < LogStash::Inputs::Base config :file_sort_direction, :validate => ["asc", "desc"], :default => "asc" public + + class << self + alias_method :old_validate_value, :validate_value + + def validate_value(value, validator) + if validator.is_a?(Array) && validator.size == 2 && validator.first.respond_to?(:call) + callable, units, val = *validator, value + validation_errors = callable.call(value, units){|coerced| val = coerced} + return validation_errors.nil? ? [true, val] : [false, validation_errors] + end + old_validate_value(value, validator) + end + end + def register require "addressable/uri" require "digest/md5" diff --git a/lib/logstash/inputs/friendly_durations.rb b/lib/logstash/inputs/friendly_durations.rb new file mode 100644 index 0000000..20b0a01 --- /dev/null +++ b/lib/logstash/inputs/friendly_durations.rb @@ -0,0 +1,46 @@ +# encoding: utf-8 + +module LogStash module Inputs + module FriendlyDurations + NUMBERS_RE = /^(?\d+(\.\d+)?)\s?(?s((ec)?(ond)?)(s)?|m((in)?(ute)?)(s)?|h(our)?(s)?|d(ay)?(s)?|w(eek)?(s)?|us(ec)?(s)?|ms(ec)?(s)?)?$/ + HOURS = 3600 + DAYS = 24 * HOURS + MEGA = 10**6 + KILO = 10**3 + + def self.call(value, unit = "sec") + val_string = value.to_s.strip + result = coerce_in_seconds(val_string, unit) + return result if result.is_a?(String) + yield result + nil + end + + private + + def self.coerce_in_seconds(value, unit) + matched = NUMBERS_RE.match(value) + if matched.nil? + return "Value '#{value}' is not a valid duration string e.g. 200 usec, 250ms, 60 sec, 18h, 21.5d, 1 day, 2w, 6 weeks" + end + multiplier = matched[:units] || unit + numeric = matched[:number].to_f + case multiplier + when "m","min","mins","minute","minutes" + numeric * 60 + when "h","hour","hours" + numeric * HOURS + when "d","day","days" + numeric * DAYS + when "w","week","weeks" + numeric * 7 * DAYS + when "ms","msec","msecs" + numeric / KILO + when "us","usec","usecs" + numeric / MEGA + else + numeric + end + end + end +end end diff --git a/spec/inputs/friendly_durations_spec.rb b/spec/inputs/friendly_durations_spec.rb new file mode 100644 index 0000000..b4cc01e --- /dev/null +++ b/spec/inputs/friendly_durations_spec.rb @@ -0,0 +1,70 @@ +# encoding: utf-8 + +require "helpers/spec_helper" +require "logstash/inputs/friendly_durations" + +describe "FriendlyDurations module function call" do + context "unacceptable strings" do + it "gives an error message for 'foobar'" do + result = LogStash::Inputs::FriendlyDurations.call("foobar","sec") + expect(result).to start_with("Value 'foobar' is not a valid duration string e.g. 200 usec") + end + it "gives an error message for '5 5 days'" do + result = LogStash::Inputs::FriendlyDurations.call("5 5 days","sec") + expect(result).to start_with("Value '5 5 days' is not a valid duration string e.g. 200 usec") + end + end + + context "when a unit is not specified, a unit override will affect the result" do + it "coerces 14 to 1209600.0 as days with a block callback" do + result = LogStash::Inputs::FriendlyDurations.call(14,"d"){|value| expect(value).to eq(1209600.0)} + expect(result).to eq(nil) + end + it "coerces '30' to 1800 as minutes with a block callback" do + result = LogStash::Inputs::FriendlyDurations.call("30","minutes"){|value| expect(value).to eq(1800.0)} + expect(result).to eq(nil) + end + end + + context "acceptable strings" do + [ + ["10", 10.0], + ["10.5 s", 10.5], + ["10.75 secs", 10.75], + ["11 second", 11.0], + ["10 seconds", 10.0], + ["500 ms", 0.5], + ["750.9 msec", 0.7509], + ["750.9 msecs", 0.7509], + ["750.9 us", 0.0007509], + ["750.9 usec", 0.0007509], + ["750.9 usecs", 0.0007509], + ["1.5m", 90.0], + ["2.5 m", 150.0], + ["1.25 min", 75.0], + ["1 minute", 60.0], + ["2.5 minutes", 150.0], + ["2h", 7200.0], + ["2 h", 7200.0], + ["1 hour", 3600.0], + ["1hour", 3600.0], + ["3 hours", 10800.0], + ["0.5d", 43200.0], + ["1day", 86400.0], + ["1 day", 86400.0], + ["2days", 172800.0], + ["14 days", 1209600.0], + ["1w", 604800.0], + ["1 w", 604800.0], + ["1 week", 604800.0], + ["2weeks", 1209600.0], + ["2 weeks", 1209600.0], + ["1.5 weeks", 907200.0], + ].each do |input, coerced| + it "coerces #{input.inspect.rjust(16)} to #{coerced.inspect} with a block callback" do + result = LogStash::Inputs::FriendlyDurations.call(input,"sec"){|value| expect(value).to eq(coerced)} + expect(result).to eq(nil) + end + end + end +end From 78651a8853566dca02c42c7d1984be16ba056ff7 Mon Sep 17 00:00:00 2001 From: Guy Boertje Date: Thu, 31 May 2018 10:35:58 +0100 Subject: [PATCH 2/2] redo the returned object as a Struct with a `to_a` method. --- lib/logstash/inputs/file.rb | 6 ++-- lib/logstash/inputs/friendly_durations.rb | 35 +++++++++++------------ spec/inputs/friendly_durations_spec.rb | 23 ++++++++------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/lib/logstash/inputs/file.rb b/lib/logstash/inputs/file.rb index 39b9b1f..81b04b4 100644 --- a/lib/logstash/inputs/file.rb +++ b/lib/logstash/inputs/file.rb @@ -229,9 +229,9 @@ class << self def validate_value(value, validator) if validator.is_a?(Array) && validator.size == 2 && validator.first.respond_to?(:call) - callable, units, val = *validator, value - validation_errors = callable.call(value, units){|coerced| val = coerced} - return validation_errors.nil? ? [true, val] : [false, validation_errors] + callable, units = *validator + # returns a ValidatedStruct having a `to_a` method suitable to return to the config mixin caller + return callable.call(value, units).to_a end old_validate_value(value, validator) end diff --git a/lib/logstash/inputs/friendly_durations.rb b/lib/logstash/inputs/friendly_durations.rb index 20b0a01..07d0ee7 100644 --- a/lib/logstash/inputs/friendly_durations.rb +++ b/lib/logstash/inputs/friendly_durations.rb @@ -8,38 +8,37 @@ module FriendlyDurations MEGA = 10**6 KILO = 10**3 - def self.call(value, unit = "sec") - val_string = value.to_s.strip - result = coerce_in_seconds(val_string, unit) - return result if result.is_a?(String) - yield result - nil + ValidatedStruct = Struct.new(:value, :error_message) do + def to_a + error_message.nil? ? [true, value] : [false, error_message] + end end - private - - def self.coerce_in_seconds(value, unit) - matched = NUMBERS_RE.match(value) + def self.call(value, unit = "sec") + # coerce into seconds + val_string = value.to_s.strip + matched = NUMBERS_RE.match(val_string) if matched.nil? - return "Value '#{value}' is not a valid duration string e.g. 200 usec, 250ms, 60 sec, 18h, 21.5d, 1 day, 2w, 6 weeks" + failed_message = "Value '#{val_string}' is not a valid duration string e.g. 200 usec, 250ms, 60 sec, 18h, 21.5d, 1 day, 2w, 6 weeks" + return ValidatedStruct.new(nil, failed_message) end multiplier = matched[:units] || unit numeric = matched[:number].to_f case multiplier when "m","min","mins","minute","minutes" - numeric * 60 + ValidatedStruct.new(numeric * 60, nil) when "h","hour","hours" - numeric * HOURS + ValidatedStruct.new(numeric * HOURS, nil) when "d","day","days" - numeric * DAYS + ValidatedStruct.new(numeric * DAYS, nil) when "w","week","weeks" - numeric * 7 * DAYS + ValidatedStruct.new(numeric * 7 * DAYS, nil) when "ms","msec","msecs" - numeric / KILO + ValidatedStruct.new(numeric / KILO, nil) when "us","usec","usecs" - numeric / MEGA + ValidatedStruct.new(numeric / MEGA, nil) else - numeric + ValidatedStruct.new(numeric, nil) end end end diff --git a/spec/inputs/friendly_durations_spec.rb b/spec/inputs/friendly_durations_spec.rb index b4cc01e..586bd86 100644 --- a/spec/inputs/friendly_durations_spec.rb +++ b/spec/inputs/friendly_durations_spec.rb @@ -7,22 +7,23 @@ context "unacceptable strings" do it "gives an error message for 'foobar'" do result = LogStash::Inputs::FriendlyDurations.call("foobar","sec") - expect(result).to start_with("Value 'foobar' is not a valid duration string e.g. 200 usec") + expect(result.error_message).to start_with("Value 'foobar' is not a valid duration string e.g. 200 usec") end it "gives an error message for '5 5 days'" do result = LogStash::Inputs::FriendlyDurations.call("5 5 days","sec") - expect(result).to start_with("Value '5 5 days' is not a valid duration string e.g. 200 usec") + expect(result.error_message).to start_with("Value '5 5 days' is not a valid duration string e.g. 200 usec") end end context "when a unit is not specified, a unit override will affect the result" do - it "coerces 14 to 1209600.0 as days with a block callback" do - result = LogStash::Inputs::FriendlyDurations.call(14,"d"){|value| expect(value).to eq(1209600.0)} - expect(result).to eq(nil) + it "coerces 14 to 1209600.0s as days" do + result = LogStash::Inputs::FriendlyDurations.call(14,"d") + expect(result.error_message).to eq(nil) + expect(result.value).to eq(1209600.0) end - it "coerces '30' to 1800 as minutes with a block callback" do - result = LogStash::Inputs::FriendlyDurations.call("30","minutes"){|value| expect(value).to eq(1800.0)} - expect(result).to eq(nil) + it "coerces '30' to 1800.0s as minutes" do + result = LogStash::Inputs::FriendlyDurations.call("30","minutes") + expect(result.to_a).to eq([true, 1800.0]) end end @@ -61,9 +62,9 @@ ["2 weeks", 1209600.0], ["1.5 weeks", 907200.0], ].each do |input, coerced| - it "coerces #{input.inspect.rjust(16)} to #{coerced.inspect} with a block callback" do - result = LogStash::Inputs::FriendlyDurations.call(input,"sec"){|value| expect(value).to eq(coerced)} - expect(result).to eq(nil) + it "coerces #{input.inspect.rjust(16)} to #{coerced.inspect}" do + result = LogStash::Inputs::FriendlyDurations.call(input,"sec") + expect(result.to_a).to eq([true, coerced]) end end end