[8.17] Upgrade elasticsearch-ruby client. (backport #17161) (backport #17306) (#17339)

* [8.x] Upgrade elasticsearch-ruby client. (backport #17161) (#17306)

* Upgrade elasticsearch-ruby client. (#17161)

* Fix Faraday removed basic auth option and apply the ES client module name change.

(cherry picked from commit e748488)

* Apply the required changes in elasticsearch_client.rb after upgrading the elasticsearch-ruby client to 8.x

* Swallow the exception and make non-connectable client when ES client raises connection refuses exception.

---------

Co-authored-by: Mashhur <[email protected]>
Co-authored-by: Mashhur <[email protected]>
(cherry picked from commit 7f74ce3)

* Update Gemfile lock to reflect elasticsearch-ruby changes.

* Upgrade faraday to v2 in Gemfile lock.

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Mashhur <[email protected]>

Author: mergify[bot]

Gemfile.jruby-3.1.lock.release

@@ -11,7 +11,7 @@ PATH
       clamp (~> 1)
       concurrent-ruby (~> 1, < 1.1.10)
       down (~> 5.2.0)
-      elasticsearch (~> 7)
+      elasticsearch (~> 8)
       filesize (~> 0.2)
       gems (~> 1)
       i18n (~> 1)
@@ -112,44 +112,23 @@ GEM
     elastic-enterprise-search (8.9.0)
       elastic-transport (~> 8.1)
       jwt (>= 1.5, < 3.0)
-    elastic-transport (8.3.5)
+    elastic-transport (8.4.0)
       faraday (< 3)
       multi_json
-    elasticsearch (7.17.11)
-      elasticsearch-api (= 7.17.11)
-      elasticsearch-transport (= 7.17.11)
-    elasticsearch-api (7.17.11)
-      multi_json
-    elasticsearch-transport (7.17.11)
-      base64
-      faraday (>= 1, < 3)
+    elasticsearch (8.17.1)
+      elastic-transport (~> 8.3)
+      elasticsearch-api (= 8.17.1)
+    elasticsearch-api (8.17.1)
       multi_json
     equalizer (0.0.11)
     et-orbi (1.2.11)
       tzinfo
-    faraday (1.10.4)
-      faraday-em_http (~> 1.0)
-      faraday-em_synchrony (~> 1.0)
-      faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0)
-      faraday-multipart (~> 1.0)
-      faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.0)
-      faraday-patron (~> 1.0)
-      faraday-rack (~> 1.0)
-      faraday-retry (~> 1.0)
-      ruby2_keywords (>= 0.0.4)
-    faraday-em_http (1.0.0)
-    faraday-em_synchrony (1.0.0)
-    faraday-excon (1.1.0)
-    faraday-httpclient (1.0.1)
-    faraday-multipart (1.0.4)
-      multipart-post (~> 2)
-    faraday-net_http (1.0.2)
-    faraday-net_http_persistent (1.2.0)
-    faraday-patron (1.0.0)
-    faraday-rack (1.0.0)
-    faraday-retry (1.0.3)
+    faraday (2.12.2)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-net_http (3.4.0)
+      net-http (>= 0.5.0)
     ffi (1.17.1-java)
     filesize (0.2.0)
     fileutils (1.7.3)
@@ -728,6 +707,8 @@ GEM
     mustermann (3.0.3)
       ruby2_keywords (~> 0.0.1)
     naught (1.1.0)
+    net-http (0.6.0)
+      uri
     net-imap (0.5.6)
       date
       net-protocol
@@ -866,6 +847,7 @@ GEM
     tzinfo-data (1.2024.2)
       tzinfo (>= 1.0.0)
     unicode-display_width (2.6.0)
+    uri (1.0.3)
     webhdfs (0.11.0)
       addressable
     webmock (3.24.0)
@@ -886,7 +868,6 @@ DEPENDENCIES
   childprocess (~> 4)
   ci_reporter_rspec (~> 1)
   date (= 3.3.3)
-  faraday (~> 1)
   fileutils (~> 1.7)
   flores (~> 0.0.8)
   fpm (~> 1, >= 1.14.1)

Gemfile.template

@@ -13,7 +13,6 @@ gem "ruby-maven-libs", "~> 3", ">= 3.9.6.1"
 gem "logstash-output-elasticsearch", ">= 11.14.0"
 gem "polyglot", require: false
 gem "treetop", require: false
-gem "faraday", "~> 1", :require => false # due elasticsearch-transport (elastic-transport) depending faraday '~> 1'
 gem "minitar", "~> 1", :group => :build
 gem "childprocess", "~> 4", :group => :build
 gem "fpm", "~> 1", ">= 1.14.1", :group => :build # compound due to bugfix https://github.com/jordansissel/fpm/pull/1856

logstash-core/lib/logstash/elasticsearch_client.rb

@@ -16,15 +16,15 @@
 # under the License.
 
 require "elasticsearch"
-require "elasticsearch/transport/transport/http/manticore"
+require "elastic/transport/transport/http/manticore"
 require 'logstash/util/manticore_ssl_config_helper'
 require 'logstash/util/password'
 
 module LogStash class ElasticsearchClient
   include LogStash::Util::Loggable
 
   class Response
-    # duplicated here from Elasticsearch::Transport::Transport::Response
+    # duplicated here from Elastic::Transport::Transport::Response
     # to create a normalised response across different client IMPL
     attr_reader :status, :body, :headers
 
@@ -65,8 +65,13 @@ def initialize(settings, logger)
     def can_connect?
       begin
         head(SecureRandom.hex(32).prepend('_'))
-      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+      rescue Elastic::Transport::Transport::Errors::BadRequest
         true
+      rescue Elastic::Transport::Transport::Errors::Unauthorized
+        true
+      rescue Exception => e
+        return true if e.message.include?('Connection refused')
+        raise e
       rescue Manticore::SocketException
         false
       end
@@ -116,7 +121,7 @@ def normalize_response(response)
 
     def client_args
       {
-        :transport_class => Elasticsearch::Transport::Transport::HTTP::Manticore,
+        :transport_class => Elastic::Transport::Transport::HTTP::Manticore,
         :hosts => [*unpack_hosts],
         # :logger => @logger, # silence the client logging
       }

logstash-core/logstash-core.gemspec

@@ -39,7 +39,7 @@ Gem::Specification.new do |gem|
   gem.email         = ["[email protected]"]
   gem.description   = %q{The core components of logstash, the scalable log and event management tool}
   gem.summary       = %q{logstash-core - The core components of logstash}
-  gem.homepage      = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  gem.homepage      = "https://www.elastic.co/logstash"
   gem.license       = "Apache-2.0"
 
   gem.files         = Dir.glob(
@@ -78,7 +78,7 @@ Gem::Specification.new do |gem|
 
   gem.add_runtime_dependency "jrjackson", "= #{ALL_VERSIONS.fetch('jrjackson')}" #(Apache 2.0 license)
 
-  gem.add_runtime_dependency "elasticsearch", '~> 7'
+  gem.add_runtime_dependency "elasticsearch", '~> 8'
   gem.add_runtime_dependency "manticore", '~> 0.6'
 
   # xpack geoip database service

logstash-core/spec/logstash/webserver_spec.rb

@@ -189,7 +189,7 @@ def free_ports(servers)
     context "and invalid basic auth is provided" do
       it 'emits an HTTP 401 with WWW-Authenticate header' do
         response = Faraday.new("http://#{api_host}:#{webserver.port}") do |conn|
-          conn.request :basic_auth, 'john-doe', 'open-sesame'
+          conn.request :authorization, :basic, 'john-doe', 'open-sesame'
         end.get('/')
         aggregate_failures do
           expect(response.status).to eq(401)
@@ -200,7 +200,7 @@ def free_ports(servers)
     context "and valid auth is provided" do
       it "returns a relevant response" do
         response = Faraday.new("http://#{api_host}:#{webserver.port}") do |conn|
-          conn.request :basic_auth, 'a-user', 's3cur3dPas!'
+          conn.request :authorization, :basic, 'a-user', 's3cur3dPas!'
         end.get('/')
         aggregate_failures do
           expect(response.status).to eq(200)

qa/integration/specs/dlq_spec.rb

@@ -83,10 +83,10 @@
       try(60) do
         begin
           result = es_client.search(index: 'test-index', size: 0, q: '*')
-        rescue Elasticsearch::Transport::Transport::Errors::ServiceUnavailable => e
+        rescue Elastic::Transport::Transport::Errors::ServiceUnavailable => e
           puts "Elasticsearch unavailable #{e.inspect}"
           hits = 0
-        rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+        rescue Elastic::Transport::Transport::Errors::NotFound => e
           puts "Index not found"
           hits = 0
         end

x-pack/qa/integration/management/multiple_pipelines_spec.rb

@@ -137,7 +137,7 @@ def count_hashes(pipelines)
 
     begin
       res = elasticsearch_client.search(index: '.monitoring-logstash-*', body: query)
-    rescue Elasticsearch::Transport::Transport::Errors::ServiceUnavailable
+    rescue Elastic::Transport::Transport::Errors::ServiceUnavailable
       return nil
     end
 

x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb

@@ -19,8 +19,8 @@
   let(:retryable_errors) do
     [NoMethodError,
      RSpec::Expectations::ExpectationNotMetError,
-     Elasticsearch::Transport::Transport::Errors::ServiceUnavailable,
-     Elasticsearch::Transport::Transport::Errors::NotFound]
+     Elastic::Transport::Transport::Errors::ServiceUnavailable,
+     Elastic::Transport::Transport::Errors::NotFound]
   end
 
   describe "metrics" do

x-pack/qa/integration/support/helpers.rb

@@ -124,7 +124,7 @@ def cleanup_system_indices(pipeline_ids)
     pipeline_ids.each do |id|
       begin
         elasticsearch_client.perform_request(:delete, "_logstash/pipeline/#{id}")
-      rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
+      rescue Elastic::Transport::Transport::Errors::NotFound => e
         puts ".logstash can be empty #{e.message}"
       end
     end

x-pack/qa/integration/support/shared_examples.rb

@@ -9,8 +9,8 @@
   let(:retryable_errors) do
     [NoMethodError,
      RSpec::Expectations::ExpectationNotMetError,
-     Elasticsearch::Transport::Transport::Errors::ServiceUnavailable,
-     Elasticsearch::Transport::Transport::Errors::NotFound]
+     Elastic::Transport::Transport::Errors::ServiceUnavailable,
+     Elastic::Transport::Transport::Errors::NotFound]
   end
 
   describe "metrics" do