Add string based time duration support. (#194)

* add string based time duration support.

* redo the returned object as a Struct with a `to_a` method.

Fixes #187

Author: Guy Boertje

Diff for: lib/logstash/inputs/file.rb

@@ -11,6 +11,7 @@
 require_relative "file_listener"
 require_relative "delete_completed_file_handler"
 require_relative "log_completed_file_handler"
+require_relative "friendly_durations"
 require "filewatch/bootstrap"
 
 # Stream events from files, normally by tailing them in a manner
@@ -109,7 +110,7 @@ class File < LogStash::Inputs::Base
   # How often (in seconds) we stat files to see if they have been modified.
   # Increasing this interval will decrease the number of system calls we make,
   # but increase the time to detect new log lines.
-  config :stat_interval, :validate => :number, :default => 1
+  config :stat_interval, :validate => [FriendlyDurations, "seconds"], :default => 1
 
   # How often (in seconds) we expand the filename patterns in the
   # `path` option to discover new files to watch.
@@ -123,7 +124,7 @@ class File < LogStash::Inputs::Base
 
   # How often (in seconds) to write a since database with the current position of
   # monitored log files.
-  config :sincedb_write_interval, :validate => :number, :default => 15
+  config :sincedb_write_interval, :validate => [FriendlyDurations, "seconds"], :default => 15
 
   # Choose where Logstash starts initially reading files: at the beginning or
   # at the end. The default behavior treats files like live streams and thus
@@ -145,7 +146,7 @@ class File < LogStash::Inputs::Base
   # After its discovery, if an ignored file is modified it is no
   # longer ignored and any new data is read. By default, this option is
   # disabled. Note this unit is in seconds.
-  config :ignore_older, :validate => :number
+  config :ignore_older, :validate => [FriendlyDurations, "seconds"]
 
   # The file input closes any files that were last read the specified
   # timespan in seconds ago.
@@ -154,7 +155,7 @@ class File < LogStash::Inputs::Base
   # reopening when new data is detected. If reading, the file will be closed
   # after closed_older seconds from when the last bytes were read.
   # The default is 1 hour
-  config :close_older, :validate => :number, :default => 1 * 60 * 60
+  config :close_older, :validate => [FriendlyDurations, "seconds"], :default => "1 hour"
 
   # What is the maximum number of file_handles that this input consumes
   # at any one time. Use close_older to close some files if you need to
@@ -191,7 +192,7 @@ class File < LogStash::Inputs::Base
   # If no changes are detected in tracked files in the last N days their sincedb
   # tracking record will expire and not be persisted.
   # This option protects against the well known inode recycling problem. (add reference)
-  config :sincedb_clean_after, :validate => :number, :default => 14 # days
+  config :sincedb_clean_after, :validate => [FriendlyDurations, "days"], :default => "14 days" # days
 
   # File content is read off disk in blocks or chunks, then using whatever the set delimiter
   # is, lines are extracted from the chunk. Specify the size in bytes of each chunk.
@@ -222,6 +223,20 @@ class File < LogStash::Inputs::Base
   config :file_sort_direction, :validate => ["asc", "desc"], :default => "asc"
 
   public
+
+  class << self
+    alias_method :old_validate_value, :validate_value
+
+    def validate_value(value, validator)
+      if validator.is_a?(Array) && validator.size == 2 && validator.first.respond_to?(:call)
+        callable, units = *validator
+        # returns a ValidatedStruct having a `to_a` method suitable to return to the config mixin caller
+        return callable.call(value, units).to_a
+      end
+      old_validate_value(value, validator)
+    end
+  end
+
   def register
     require "addressable/uri"
     require "digest/md5"

Diff for: lib/logstash/inputs/friendly_durations.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+module LogStash module Inputs
+  module FriendlyDurations
+    NUMBERS_RE = /^(?<number>\d+(\.\d+)?)\s?(?<units>s((ec)?(ond)?)(s)?|m((in)?(ute)?)(s)?|h(our)?(s)?|d(ay)?(s)?|w(eek)?(s)?|us(ec)?(s)?|ms(ec)?(s)?)?$/
+    HOURS = 3600
+    DAYS = 24 * HOURS
+    MEGA = 10**6
+    KILO = 10**3
+
+    ValidatedStruct = Struct.new(:value, :error_message) do
+      def to_a
+        error_message.nil? ? [true, value] : [false, error_message]
+      end
+    end
+
+    def self.call(value, unit = "sec")
+      # coerce into seconds
+      val_string = value.to_s.strip
+      matched = NUMBERS_RE.match(val_string)
+      if matched.nil?
+        failed_message = "Value '#{val_string}' is not a valid duration string e.g. 200 usec, 250ms, 60 sec, 18h, 21.5d, 1 day, 2w, 6 weeks"
+        return ValidatedStruct.new(nil, failed_message)
+      end
+      multiplier = matched[:units] || unit
+      numeric = matched[:number].to_f
+      case multiplier
+      when "m","min","mins","minute","minutes"
+        ValidatedStruct.new(numeric * 60, nil)
+      when "h","hour","hours"
+        ValidatedStruct.new(numeric * HOURS, nil)
+      when "d","day","days"
+        ValidatedStruct.new(numeric * DAYS, nil)
+      when "w","week","weeks"
+        ValidatedStruct.new(numeric * 7 * DAYS, nil)
+      when "ms","msec","msecs"
+        ValidatedStruct.new(numeric / KILO, nil)
+      when "us","usec","usecs"
+        ValidatedStruct.new(numeric / MEGA, nil)
+      else
+        ValidatedStruct.new(numeric, nil)
+      end
+    end
+  end
+end end

Diff for: spec/inputs/friendly_durations_spec.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+
+require "helpers/spec_helper"
+require "logstash/inputs/friendly_durations"
+
+describe "FriendlyDurations module function call" do
+  context "unacceptable strings" do
+    it "gives an error message for 'foobar'" do
+      result = LogStash::Inputs::FriendlyDurations.call("foobar","sec")
+      expect(result.error_message).to start_with("Value 'foobar' is not a valid duration string e.g. 200 usec")
+    end
+    it "gives an error message for '5 5 days'" do
+      result = LogStash::Inputs::FriendlyDurations.call("5 5 days","sec")
+      expect(result.error_message).to start_with("Value '5 5 days' is not a valid duration string e.g. 200 usec")
+    end
+  end
+
+  context "when a unit is not specified, a unit override will affect the result" do
+    it "coerces 14 to 1209600.0s as days" do
+      result = LogStash::Inputs::FriendlyDurations.call(14,"d")
+      expect(result.error_message).to eq(nil)
+      expect(result.value).to eq(1209600.0)
+    end
+    it "coerces '30' to 1800.0s as minutes" do
+      result = LogStash::Inputs::FriendlyDurations.call("30","minutes")
+      expect(result.to_a).to eq([true, 1800.0])
+    end
+  end
+
+  context "acceptable strings" do
+    [
+      ["10",                10.0],
+      ["10.5 s",            10.5],
+      ["10.75 secs",        10.75],
+      ["11 second",         11.0],
+      ["10 seconds",        10.0],
+      ["500 ms",             0.5],
+      ["750.9 msec",         0.7509],
+      ["750.9 msecs",        0.7509],
+      ["750.9 us",           0.0007509],
+      ["750.9 usec",         0.0007509],
+      ["750.9 usecs",        0.0007509],
+      ["1.5m",              90.0],
+      ["2.5 m",            150.0],
+      ["1.25 min",          75.0],
+      ["1 minute",          60.0],
+      ["2.5 minutes",      150.0],
+      ["2h",              7200.0],
+      ["2 h",             7200.0],
+      ["1 hour",          3600.0],
+      ["1hour",           3600.0],
+      ["3 hours",        10800.0],
+      ["0.5d",           43200.0],
+      ["1day",           86400.0],
+      ["1 day",          86400.0],
+      ["2days",         172800.0],
+      ["14 days",      1209600.0],
+      ["1w",            604800.0],
+      ["1 w",           604800.0],
+      ["1 week",        604800.0],
+      ["2weeks",       1209600.0],
+      ["2 weeks",      1209600.0],
+      ["1.5 weeks",     907200.0],
+    ].each do |input, coerced|
+      it "coerces #{input.inspect.rjust(16)} to #{coerced.inspect}" do
+        result = LogStash::Inputs::FriendlyDurations.call(input,"sec")
+        expect(result.to_a).to eq([true, coerced])
+      end
+    end
+  end
+end