Added in parameter enabled timeframe support

docs/index.asciidoc

@@ -117,6 +117,11 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-schedule>> |<<hash,hash>>|Yes
 | <<plugins-{type}s-{plugin}-socket_timeout>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-target>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-time_back_buffer_string>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-time_back_buffer>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-time_forward_buffer_string>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-time_forward_buffer>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-time_format>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-truststore_type>> |<<string,string>>|No
@@ -324,6 +329,51 @@ Timeout (in seconds) to wait for data on the socket. Default is `10s`
 
 Define the target field for placing the received data. If this setting is omitted, the data will be stored at the root (top level) of the event.
 
+[id="plugins-{type}s-{plugin}-time_back_buffer_string"]
+===== `time_back_buffer_string`
+
+  * Value type is <<string,string>>
+  * Default value is `time_back_buffer`
+
+The text to replace in the URL with a formatted time string that represents a time in the past
+
+[id="plugins-{type}s-{plugin}-time_back_buffer"]
+===== `time_back_buffer`
+
+  * Value type is <<number,number>>
+  * There is no default value for this setting
+
+The time in seconds to set the 'time_back_buffer' backwards
+
+[id="plugins-{type}s-{plugin}-time_forward_buffer_string"]
+===== `time_forward_buffer_string`
+
+  * Value type is <<string,string>>
+  * Default value is `time_forward_buffer`
+
+The text to replace in the URL with a formatted time string that represents a time in the future
+
+[id="plugins-{type}s-{plugin}-time_forward_buffer"]
+===== `time_forward_buffer`
+
+  * Value type is <<number,number>>
+  * Default value is `0`
+
+The time in seconds to set the 'time_forward_buffer' forwards
+
+[id="plugins-{type}s-{plugin}-time_format"]
+===== `time_format`
+
+  * Value type is <<string,string>>
+  * Default value is `'%s'`
+
+The string representing the ruby DateTime strftime output format.
+Examples include:
+  1. %FT%R    - 2007-11-19T08:37            Calendar date and local time (extended)
+  2. %FT%T%:z - 2007-11-19T08:37:48-06:00   Date and time of day for calendar date (extended)
+  3. %s       - Number of seconds since 1970-01-01 00:00:00 UTC.
+  4. %Q       - Number of milliseconds since 1970-01-01 00:00:00 UTC.
+
 [id="plugins-{type}s-{plugin}-truststore"]
 ===== `truststore` 
 

lib/logstash/inputs/http_poller.rb

@@ -5,6 +5,7 @@
 require "socket" # for Socket.gethostname
 require "manticore"
 require "rufus/scheduler"
+require "date"
 
 class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   include LogStash::PluginMixins::HttpClient
@@ -35,6 +36,28 @@ class LogStash::Inputs::HTTP_Poller < LogStash::Inputs::Base
   # hash of metadata.
   config :metadata_target, :validate => :string, :default => '@metadata'
 
+  # The name of a variable to use in string replacement for time based calls in the past
+  # making available as a variable to work around hard-coded string substition issues
+  config :time_back_buffer_string, :validate => :string, :default => 'time_back_buffer'
+
+  # The amount of time in seconds to poll backwards
+  config :time_back_buffer, :validate => :number
+
+  # The name of a variable to use in string replacement for time based calls in the past
+  # making available as a variable to work around hard-coded string substition issues
+  config :time_forward_buffer_string, :validate => :string, :default => 'time_forward_buffer'
+
+  # The amount of time in seconds to poll forwards
+  config :time_forward_buffer, :validate => :number, :default => 0
+
+  # get the timeformat, to support seconds and milliseconds
+  # Common time formats and codes
+  # %FT%R     - 2007-11-19T08:37          Calendar date and local time (extended)
+  # %FT%T%:z  - 2007-11-19T08:37:48-06:00 Date and time of day for calendar date (extended)
+  # %s      - Number of seconds since 1970-01-01 00:00:00 UTC.
+  # %Q      - Number of milliseconds since 1970-01-01 00:00:00 UTC.
+  config :time_format, :validate => :string, :default => '%s'
+
   public
   Schedule_types = %w(cron every at in)
   def register
@@ -147,13 +170,116 @@ def run_once(queue)
   private
   def request_async(queue, name, request)
     @logger.debug? && @logger.debug("Fetching URL", :name => name, :url => request)
+    @logger.debug? && @logger.debug("Forward Buffer", :buffer => @time_forward_buffer)
+    @logger.debug? && @logger.debug("Backward Buffer", :buffer => @time_back_buffer)
+    @logger.debug? && @logger.debug("Time Format", :format => @time_format)
+    # Grab the current time
     started = Time.now
 
+    # this needs to be a DateTime to deal with subtractions
+    currenttime = DateTime.now
+    @logger.debug? && @logger.debug("Current Time", :currenttime => currenttime)
+    # If we have the @time_back_buffer set, we modify the URL with a calculated timestamp
+    back_buffer = @time_back_buffer
+    forward_buffer = @time_forward_buffer
+
+    # To support multiple formats
+    # https://apidock.com/ruby/DateTime/strftime
+    if @time_format
+      time_format_code = @time_format
+    end
+
+    # Deal with buffers going backwards
+    if @time_back_buffer && @time_back_buffer > 0
+      # Rational is fractions, and the second number is the number of seconds in a day
+      # Datetime is the time since the unix epoch, and it works using rational numbers
+      # https://stackoverflow.com/a/10056201 has more info
+      buffer = currenttime - Rational(back_buffer,86400)
+      @logger.debug? && @logger.debug("Back Buffer", :buffer => buffer)
+      # Turns out I don't need to handle arrays, as each request is processed separately   
+      # but I'll keep the loop here, for reference
+      if request[1].include?("#{time_back_buffer_string}")
+        @logger.debug? && @logger.debug("URL timestamp - backwards - pre:", :url => request[1])
+        request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code))
+        # Store the timestamp as a variable to swap it back after the URL has been fetched
+        @buffer_time_back = buffer.strftime(time_format_code)
+        @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[1])
+      end
+      #request.each_with_index do |entry, i| 
+      #  # We need to verify we're working with a string, otherwise we run in to method not found errors 
+      #  if request[i].is_a? String 
+      #    # And lets only modify strings that actually include our text
+      #    if request[i].include?("#{time_back_buffer_string}")
+      #      # Originally request[1] = request[1].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code))
+      #      @logger.debug? && @logger.debug("URL timestamp - backwards - pre:", :url => request[i])
+      #      request[i] = request[i].gsub(/#{time_back_buffer_string}/,buffer.strftime(time_format_code))
+      #      # Store the timestamp as a variable to swap it back after the URL has been fetched
+      #      @buffer_time_back = buffer.strftime(time_format_code)
+      #      @logger.debug? && @logger.debug("URL timestamp - backwards - post:", :url => request[i])
+      #      @logger.debug? && @logger.debug("URL timestamp - backwards - string:", :time => buffer_time_back)
+      #    end
+      #  end
+      #end
+    end
+
+    # deal with forward buffers, if we need to
+    # We can tolerate a zero here because it would indicate 'now'
+    if @time_forward_buffer && @time_forward_buffer >= 0
+      # Rational is fractions, and the second number is the number of seconds in a day
+      # Datetime is the time since the unix epoch, and it works using rational numbers
+      # https://stackoverflow.com/a/10056201 has more info
+      buffer = currenttime + Rational(forward_buffer,86400)
+      @logger.debug? && @logger.debug("Forward Buffer", :buffer => buffer)
+      if request[1].include?("#{time_forward_buffer_string}")
+        @logger.debug? && @logger.debug("URL timestamp - forward - pre:", :url => request[1])
+        request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code))
+        # Store the timestamp as a variable to swap it back after the URL has been fetched
+        @buffer_time_forward = buffer.strftime(time_format_code)
+        @logger.debug? && @logger.debug("URL timestamp - forward - post:", :url => request[1])
+      end
+      #  request.each_with_index do |entry, i| 
+      #  # We need to verify we're working with a string, otherwise we run in to method not found errors 
+      #  if request[i].is_a? String 
+      #    # And lets only modify strings that actually include our text
+      #    if request[i].include?("#{time_forward_buffer_string}")
+      #      # Originally request[1] = request[1].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code))
+      #      @logger.debug? && @logger.debug("URL timestamp - forwards - pre:", :url => request[i])
+      #      request[i] = request[i].gsub(/#{time_forward_buffer_string}/,buffer.strftime(time_format_code))
+      #      # Store the timestamp as a variable to swap it back after the URL has been fetched
+      #      @buffer_time_forward = buffer.strftime(time_format_code)
+      #      @logger.debug? && @logger.debug("URL timestamp - forwards - post:", :url => request[i])
+      #      @logger.debug? && @logger.debug("URL timestamp - forwards - string:", :time => buffer_time_forward)
+      #    end
+      #  end
+      #end
+    end
+
     method, *request_opts = request
     client.async.send(method, *request_opts).
-      on_success {|response| handle_success(queue, name, request, response, Time.now - started)}.
+      on_success {|response| 
+      @logger.debug? && @logger.debug("URL timestamp - success - pre:", :url => request[1])
+      handle_success(queue, name, request, response, Time.now - started)
+      # If either of out buffers were set, replace the contents of the URL back to our string
+      # It has been observed that the URL wasn't being set back, hence this workaround
+      if @buffer_time_back
+        request[1] = request[1].gsub(@buffer_time_back,time_back_buffer_string)
+      end
+      if @buffer_time_forward
+        request[1] = request[1].gsub(@buffer_time_forward,time_forward_buffer_string)
+      end
+      @logger.debug? && @logger.debug("URL timestamp - success - post:", :url => request[1])}.
       on_failure {|exception|
+      @logger.debug? && @logger.debug("URL timestamp - failure - pre:", :url => request[1])
       handle_failure(queue, name, request, exception, Time.now - started)
+      # If either of out buffers were set, replace the contents of the URL back to our string
+      # It has been observed that the URL wasn't being set back, hence this workaround
+      if @buffer_time_back
+        request[1] = request[1].gsub(@buffer_time_back,time_back_buffer_string)
+      end
+      if @buffer_time_forward
+        request[1] = request[1].gsub(@buffer_time_forward,time_forward_buffer_string)
+      end
+      @logger.debug? && @logger.debug("URL timestamp - failure - post:", :url => request[1])
     }
   end