From 52e35eefa538aee6b0eec6f9c48f8196f4ac6722 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 29 Nov 2021 10:56:31 +0100
Subject: [PATCH 01/30] Refactor: memoize hash-es to avoid allocation

---
 lib/logstash/outputs/elasticsearch/http_client.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb
index 9efaeaf1..975e348a 100644
--- a/lib/logstash/outputs/elasticsearch/http_client.rb
+++ b/lib/logstash/outputs/elasticsearch/http_client.rb
@@ -283,11 +283,11 @@ def uris
     end
 
     def client_settings
-      @options[:client_settings] || {}
+      @_client_settings ||= @options[:client_settings] || {}
     end
 
     def ssl_options
-      client_settings.fetch(:ssl, {})
+      @_ssl_options ||= client_settings.fetch(:ssl, {})
     end
 
     def http_compression

From 5ed7a4c61d6b870913c88a4230a7f4a9ca256dc5 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 29 Nov 2021 12:18:54 +0100
Subject: [PATCH 02/30] Feat: ssl_enabled_protocols support (draft)

---
 lib/logstash/outputs/elasticsearch/http_client_builder.rb | 4 ++++
 lib/logstash/plugin_mixins/elasticsearch/api_configs.rb   | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/lib/logstash/outputs/elasticsearch/http_client_builder.rb b/lib/logstash/outputs/elasticsearch/http_client_builder.rb
index fb181fa5..d23ee697 100644
--- a/lib/logstash/outputs/elasticsearch/http_client_builder.rb
+++ b/lib/logstash/outputs/elasticsearch/http_client_builder.rb
@@ -132,11 +132,15 @@ def self.setup_ssl(logger, params)
         ssl_options[:keystore] = keystore
         ssl_options[:keystore_password] = keystore_password.value if keystore_password
       end
+
       if !params["ssl_certificate_verification"]
         logger.warn "You have enabled encryption but DISABLED certificate verification, " +
                     "to make sure your data is secure remove `ssl_certificate_verification => false`"
         ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname
       end
+
+      ssl_options[:protocols] = params['ssl_enabled_protocols'] if params['ssl_enabled_protocols']
+
       { ssl: ssl_options }
     end
 
diff --git a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
index 1721a0c0..bb4417b5 100644
--- a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
+++ b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
@@ -66,6 +66,8 @@ module APIConfigs
         # Set the keystore password
         :keystore_password => { :validate => :password },
 
+        :ssl_enabled_protocols => { :validate => :string, :default => ['TLSv1.2', 'TLSv1.3'], :list => true },
+
         # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
         # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
         # this with master nodes, you probably want to disable HTTP on them by setting

From bbbd02d270310a8d5169aea4940167ac09061844 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 29 Nov 2021 12:35:55 +0100
Subject: [PATCH 03/30] CI: test with
 xpack.security.http.ssl.supported_protocols

---
 .ci/Dockerfile.elasticsearch | 1 +
 .travis.yml                  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.ci/Dockerfile.elasticsearch b/.ci/Dockerfile.elasticsearch
index 8ad98bf2..9d0afd6e 100644
--- a/.ci/Dockerfile.elasticsearch
+++ b/.ci/Dockerfile.elasticsearch
@@ -25,6 +25,7 @@ RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then \
       fi \
     fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi
+RUN if [ "$SECURE_INTEGRATION" = "true" ] && [ ! -z "$ES_SSL_SUPPORTED_PROTOCOLS" ] ; then echo "xpack.security.http.ssl.supported_protocols: ${ES_SSL_SUPPORTED_PROTOCOLS}" >> $es_yml; fi
 
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser; fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser; fi
diff --git a/.travis.yml b/.travis.yml
index 6534b863..9eeaa6c8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,3 +12,4 @@ env:
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_KEY_INVALID=true LOG_LEVEL=info
+  - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3 LOG_LEVEL=info

From b41c9553e50d8a2942620fcd48d6d27cff11851e Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 29 Nov 2021 12:40:06 +0100
Subject: [PATCH 04/30] Test: adjust a test expectation (new protocols: ...)

---
 spec/unit/outputs/elasticsearch_ssl_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/unit/outputs/elasticsearch_ssl_spec.rb b/spec/unit/outputs/elasticsearch_ssl_spec.rb
index 3186eefc..b7b9857c 100644
--- a/spec/unit/outputs/elasticsearch_ssl_spec.rb
+++ b/spec/unit/outputs/elasticsearch_ssl_spec.rb
@@ -33,7 +33,7 @@
     
     it "should pass the flag to the ES client" do
       expect(::Manticore::Client).to receive(:new) do |args|
-        expect(args[:ssl]).to eq(:enabled => true, :verify => :disable)
+        expect(args[:ssl]).to match hash_including(:enabled => true, :verify => :disable)
       end.and_return(manticore_double)
       
       subject.register

From c17bac95b7776340a390990d0e5e4d7720fc7c87 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 29 Nov 2021 13:28:40 +0100
Subject: [PATCH 05/30] Test: integration spec with TLSv1.2 vs TLSv1.3

---
 spec/integration/outputs/index_spec.rb | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index 6e817052..2f562bce 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -156,7 +156,7 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
         "cacert" => cacert,
         "index" => index
       }
-    end
+    end 
 
     let(:curl_opts) { "-u #{user}:#{password}" }
 
@@ -219,6 +219,29 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
         include_examples("an indexer", true)
       end
 
+      context 'with enforced TLSv1.3 protocol' do
+        let(:config) { super().merge 'ssl_enabled_protocols' => [ 'TLSv1.3' ] }
+
+        it_behaves_like("an indexer", true)
+      end
+
+      context 'with enforced TLSv1.2 protocol (while ES only enabled TLSv1.3)' do
+        let(:config) { super().merge 'ssl_enabled_protocols' => [ 'TLSv1.2' ] }
+
+        it "does not ship events" do
+          subject.multi_receive(events)
+
+          http_client.post("#{es_url}/_refresh").call
+
+          response = http_client.get("#{index_url}/_count?q=*")
+          result = LogStash::Json.load(response.body)
+          cur_count = result["count"]
+          expect(cur_count).to eq(0) # ES output keeps re-trying but ends up with a
+          # [Manticore::ClientProtocolException] Received fatal alert: protocol_version
+        end
+
+      end if ENV['ES_SSL_SUPPORTED_PROTOCOLS'] == 'TLSv1.3'
+
     end
 
   end

From e8d0abd426fa4021417b2dc5780ce37e5371a56a Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 29 Nov 2021 16:22:49 +0100
Subject: [PATCH 06/30] CI: pass down $ES_SSL_SUPPORTED_PROTOCOLS

---
 .ci/Dockerfile.elasticsearch    | 3 +++
 .ci/docker-compose.override.yml | 4 ++++
 .ci/logstash-run.sh             | 3 +++
 3 files changed, 10 insertions(+)

diff --git a/.ci/Dockerfile.elasticsearch b/.ci/Dockerfile.elasticsearch
index 9d0afd6e..8aab4d4c 100644
--- a/.ci/Dockerfile.elasticsearch
+++ b/.ci/Dockerfile.elasticsearch
@@ -6,6 +6,7 @@ ARG es_path=/usr/share/elasticsearch
 ARG es_yml=$es_path/config/elasticsearch.yml
 ARG SECURE_INTEGRATION
 ARG ES_SSL_KEY_INVALID
+ARG ES_SSL_SUPPORTED_PROTOCOLS
 
 RUN rm -f $es_path/config/scripts
 
@@ -27,5 +28,7 @@ RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then \
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] && [ ! -z "$ES_SSL_SUPPORTED_PROTOCOLS" ] ; then echo "xpack.security.http.ssl.supported_protocols: ${ES_SSL_SUPPORTED_PROTOCOLS}" >> $es_yml; fi
 
+RUN cat $es_yml
+
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser; fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser; fi
diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml
index 9812f568..260b0e50 100644
--- a/.ci/docker-compose.override.yml
+++ b/.ci/docker-compose.override.yml
@@ -12,6 +12,7 @@ services:
         - INTEGRATION=${INTEGRATION:-false}
         - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
         - ES_SSL_KEY_INVALID=${ES_SSL_KEY_INVALID:-false}
+        - ES_SSL_SUPPORTED_PROTOCOLS=$ES_SSL_SUPPORTED_PROTOCOLS
 
   elasticsearch:
     build:
@@ -22,6 +23,9 @@ services:
         - INTEGRATION=${INTEGRATION:-false}
         - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
         - ES_SSL_KEY_INVALID=${ES_SSL_KEY_INVALID:-false}
+        - ES_SSL_SUPPORTED_PROTOCOLS=$ES_SSL_SUPPORTED_PROTOCOLS
+    environment:
+      - ES_JAVA_OPTS=-Xms640m -Xmx640m
     command: /usr/share/elasticsearch/elasticsearch-run.sh
     tty: true
     ports:
diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh
index 990e0503..09da8ac0 100755
--- a/.ci/logstash-run.sh
+++ b/.ci/logstash-run.sh
@@ -1,4 +1,7 @@
 #!/bin/bash
+
+env
+
 set -ex
 
 export PATH=$BUILD_DIR/gradle/bin:$PATH

From 90736a6d30c4a7e5f63942aae51623afa3fb9e39 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Tue, 30 Nov 2021 18:54:32 +0100
Subject: [PATCH 07/30] Temp: debug LS run

---
 .ci/docker-compose.override.yml | 1 +
 .ci/logstash-run.sh             | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml
index 260b0e50..9ded02e3 100644
--- a/.ci/docker-compose.override.yml
+++ b/.ci/docker-compose.override.yml
@@ -13,6 +13,7 @@ services:
         - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
         - ES_SSL_KEY_INVALID=${ES_SSL_KEY_INVALID:-false}
         - ES_SSL_SUPPORTED_PROTOCOLS=$ES_SSL_SUPPORTED_PROTOCOLS
+        - LOG_LEVEL=debug
 
   elasticsearch:
     build:
diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh
index 09da8ac0..07a4fd31 100755
--- a/.ci/logstash-run.sh
+++ b/.ci/logstash-run.sh
@@ -14,7 +14,7 @@ fi
 
 wait_for_es() {
   count=120
-  while ! curl -s $ES_URL >/dev/null && [[ $count -ne 0 ]]; do
+  while ! curl -vik $ES_URL >/dev/null && [[ $count -ne 0 ]]; do
     count=$(( $count - 1 ))
     [[ $count -eq 0 ]] && exit 1
     sleep 1

From eca80e632c019979c18872041335b54fadf6b813 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Tue, 30 Nov 2021 19:18:26 +0100
Subject: [PATCH 08/30] Temp: second service

---
 .ci/logstash-run.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh
index 07a4fd31..4285526f 100755
--- a/.ci/logstash-run.sh
+++ b/.ci/logstash-run.sh
@@ -14,12 +14,12 @@ fi
 
 wait_for_es() {
   count=120
-  while ! curl -vik $ES_URL >/dev/null && [[ $count -ne 0 ]]; do
+  while ! curl --tlsv1.2 --tls-max 1.3 -vik $ES_URL >/dev/null && [[ $count -ne 0 ]]; do
     count=$(( $count - 1 ))
     [[ $count -eq 0 ]] && exit 1
     sleep 1
   done
-  echo $(curl -s $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
+  echo $(curl --tlsv1.2 --tls-max 1.3 -s $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
 }
 
 if [[ "$INTEGRATION" != "true" ]]; then

From 5705182a29529a12485d10c20f7230fbaa4d501f Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 1 Dec 2021 10:52:44 +0100
Subject: [PATCH 09/30] restore running with CURL_OPTS

---
 .ci/logstash-run.sh | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/.ci/logstash-run.sh b/.ci/logstash-run.sh
index 4285526f..3e7bd084 100755
--- a/.ci/logstash-run.sh
+++ b/.ci/logstash-run.sh
@@ -7,23 +7,26 @@ set -ex
 export PATH=$BUILD_DIR/gradle/bin:$PATH
 
 if [[ "$SECURE_INTEGRATION" == "true" ]]; then
-  ES_URL="https://elasticsearch:9200 -k"
+  ES_URL="https://elasticsearch:9200"
 else
   ES_URL="http://elasticsearch:9200"
 fi
 
+# CentOS 7 using curl defaults does not enable TLSv1.3
+CURL_OPTS="-k --tlsv1.2 --tls-max 1.3"
+
 wait_for_es() {
   count=120
-  while ! curl --tlsv1.2 --tls-max 1.3 -vik $ES_URL >/dev/null && [[ $count -ne 0 ]]; do
+  while ! curl $CURL_OPTS $ES_URL >/dev/null && [[ $count -ne 0 ]]; do
     count=$(( $count - 1 ))
     [[ $count -eq 0 ]] && exit 1
     sleep 1
   done
-  echo $(curl --tlsv1.2 --tls-max 1.3 -s $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
+  echo $(curl $CURL_OPTS -vi $ES_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
 }
 
 if [[ "$INTEGRATION" != "true" ]]; then
-  bundle exec rspec -fd spec/unit -t ~integration -t ~secure_integration
+  jruby -rbundler/setup -S rspec -fd spec/unit -t ~integration -t ~secure_integration
 else
 
   if [[ "$SECURE_INTEGRATION" == "true" ]]; then
@@ -35,5 +38,5 @@ else
   echo "Waiting for elasticsearch to respond..."
   ES_VERSION=$(wait_for_es)
   echo "Elasticsearch $ES_VERSION is Up!"
-  bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag es_version:$ES_VERSION spec/integration
+  jruby -rbundler/setup -S rspec -fd $extra_tag_args --tag update_tests:painless --tag es_version:$ES_VERSION spec/integration
 fi

From 90d2e6adf2ce5ffc0d29e4b9020a87ddf1c827d9 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 1 Dec 2021 12:14:12 +0100
Subject: [PATCH 10/30] Test: refactor - multi_receive might block

---
 spec/integration/outputs/index_spec.rb | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index 2f562bce..f26334f9 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -76,9 +76,12 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
 
   before do
     subject.register
-    subject.multi_receive([])
   end
-  
+
+  after do
+    subject.do_close
+  end
+
   shared_examples "an indexer" do |secure|
     it "ships events" do
       subject.multi_receive(events)
@@ -146,10 +149,10 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
     let(:user) { "simpleuser" }
     let(:password) { "abc123" }
     let(:cacert) { "spec/fixtures/test_certs/ca.crt" }
-    let(:es_url) {"https://elasticsearch:9200"}
+    let(:es_url) { "https://#{get_host_port}" }
     let(:config) do
       {
-        "hosts" => ["elasticsearch:9200"],
+        "hosts" => [ get_host_port ],
         "user" => user,
         "password" => password,
         "ssl" => true,

From a7d754e8a943368d034836d23745804bbde442bf Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 1 Dec 2021 12:23:36 +0100
Subject: [PATCH 11/30] restore back initial dummy multi_receive []

---
 spec/integration/outputs/index_spec.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index f26334f9..d7b32e64 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -76,6 +76,7 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
 
   before do
     subject.register
+    subject.multi_receive([])
   end
 
   after do

From abaa29601bf287ebcedd51c7fd8d93f7aabf90c7 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 1 Dec 2021 12:37:09 +0100
Subject: [PATCH 12/30] Test: restore TLS 1.3 retry looping spec

---
 spec/integration/outputs/index_spec.rb | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index d7b32e64..ee41e0e8 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -74,9 +74,11 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
     end
   end
 
+  let(:initial_events) { [] }
+
   before do
     subject.register
-    subject.multi_receive([])
+    subject.multi_receive(initial_events) if initial_events
   end
 
   after do
@@ -232,8 +234,11 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
       context 'with enforced TLSv1.2 protocol (while ES only enabled TLSv1.3)' do
         let(:config) { super().merge 'ssl_enabled_protocols' => [ 'TLSv1.2' ] }
 
+        let(:initial_events) { nil }
+
         it "does not ship events" do
-          subject.multi_receive(events)
+          Thread.start { subject.multi_receive(events) } # we'll be stuck in a retry loop
+          sleep 2.5
 
           http_client.post("#{es_url}/_refresh").call
 

From 73e4a2226e3d36a1ee79393d94e3cbd6763e0c8d Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 1 Dec 2021 12:55:13 +0100
Subject: [PATCH 13/30] make sure index exists

---
 spec/integration/outputs/index_spec.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index ee41e0e8..06ddf104 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -237,6 +237,7 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
         let(:initial_events) { nil }
 
         it "does not ship events" do
+          http_client.put(index_url).call # make sure index exists
           Thread.start { subject.multi_receive(events) } # we'll be stuck in a retry loop
           sleep 2.5
 

From cacdd6bbb8a180f633519e6a0aedabc27d1c4796 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 1 Dec 2021 13:16:51 +0100
Subject: [PATCH 14/30] CI: disable debug logging LS

---
 .ci/docker-compose.override.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml
index 9ded02e3..260b0e50 100644
--- a/.ci/docker-compose.override.yml
+++ b/.ci/docker-compose.override.yml
@@ -13,7 +13,6 @@ services:
         - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
         - ES_SSL_KEY_INVALID=${ES_SSL_KEY_INVALID:-false}
         - ES_SSL_SUPPORTED_PROTOCOLS=$ES_SSL_SUPPORTED_PROTOCOLS
-        - LOG_LEVEL=debug
 
   elasticsearch:
     build:

From d8e8c27739d847c0a5aabde887c1f3310aa99b2a Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 24 Jan 2022 13:10:48 +0100
Subject: [PATCH 15/30] adjust spec for using curl instead of manticore

---
 spec/integration/outputs/index_spec.rb | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index 06ddf104..813c5775 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -237,14 +237,13 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
         let(:initial_events) { nil }
 
         it "does not ship events" do
-          http_client.put(index_url).call # make sure index exists
+          curl_and_get_json_response index_url, method: :put # make sure index exists
           Thread.start { subject.multi_receive(events) } # we'll be stuck in a retry loop
           sleep 2.5
 
-          http_client.post("#{es_url}/_refresh").call
+          curl_and_get_json_response "#{es_url}/_refresh", method: :post
 
-          response = http_client.get("#{index_url}/_count?q=*")
-          result = LogStash::Json.load(response.body)
+          result = curl_and_get_json_response "#{index_url}/_count?q=*"
           cur_count = result["count"]
           expect(cur_count).to eq(0) # ES output keeps re-trying but ends up with a
           # [Manticore::ClientProtocolException] Received fatal alert: protocol_version

From ecdfa40e4e2427bfb536133ca412f0e8b18438ba Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 24 Jan 2022 13:49:33 +0100
Subject: [PATCH 16/30] Test: run with curl --tlsv1.2 --tls-max 1.3

---
 spec/integration/outputs/index_spec.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index 813c5775..fd9ec53a 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -203,6 +203,8 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
 
     else
 
+      let(:curl_opts) { "#{super()} --tlsv1.2 --tls-max 1.3" }
+
       it_behaves_like("an indexer", true)
 
       describe "with a password requiring escaping" do

From bd1ec71808eefaa2128ad7fe7b042d60a1a8cd4d Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Tue, 25 Jan 2022 18:02:37 +0100
Subject: [PATCH 17/30] rename new option to ssl_supported_protocols

---
 lib/logstash/outputs/elasticsearch/http_client_builder.rb | 2 +-
 lib/logstash/plugin_mixins/elasticsearch/api_configs.rb   | 2 +-
 spec/integration/outputs/index_spec.rb                    | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/logstash/outputs/elasticsearch/http_client_builder.rb b/lib/logstash/outputs/elasticsearch/http_client_builder.rb
index d23ee697..e575b566 100644
--- a/lib/logstash/outputs/elasticsearch/http_client_builder.rb
+++ b/lib/logstash/outputs/elasticsearch/http_client_builder.rb
@@ -139,7 +139,7 @@ def self.setup_ssl(logger, params)
         ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname
       end
 
-      ssl_options[:protocols] = params['ssl_enabled_protocols'] if params['ssl_enabled_protocols']
+      ssl_options[:protocols] = params['ssl_supported_protocols'] if params['ssl_supported_protocols']
 
       { ssl: ssl_options }
     end
diff --git a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
index bb4417b5..945cebf9 100644
--- a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
+++ b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
@@ -66,7 +66,7 @@ module APIConfigs
         # Set the keystore password
         :keystore_password => { :validate => :password },
 
-        :ssl_enabled_protocols => { :validate => :string, :default => ['TLSv1.2', 'TLSv1.3'], :list => true },
+        :ssl_supported_protocols => { :validate => :string, :default => ['TLSv1.2', 'TLSv1.3'], :list => true },
 
         # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
         # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index fd9ec53a..b0f6bc82 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -228,13 +228,13 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
       end
 
       context 'with enforced TLSv1.3 protocol' do
-        let(:config) { super().merge 'ssl_enabled_protocols' => [ 'TLSv1.3' ] }
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.3' ] }
 
         it_behaves_like("an indexer", true)
       end
 
       context 'with enforced TLSv1.2 protocol (while ES only enabled TLSv1.3)' do
-        let(:config) { super().merge 'ssl_enabled_protocols' => [ 'TLSv1.2' ] }
+        let(:config) { super().merge 'ssl_supported_protocols' => [ 'TLSv1.2' ] }
 
         let(:initial_events) { nil }
 

From 4f9ad283951d62dd5da2c7f3dee4917d478869d7 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 26 Jan 2022 09:21:27 +0100
Subject: [PATCH 18/30] bump + changelog

---
 CHANGELOG.md                          | 3 +++
 logstash-output-elasticsearch.gemspec | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1071291d..30c2ceda 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,6 @@
+## 11.5.0
+ - Feat: add ssl_supported_protocols option [#1055](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1055)
+
 ## 11.4.1
  - Feat: upgrade manticore (http-client) library [#1063](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1063)
    - the underlying changes include latest HttpClient (4.5.13)
diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec
index eb33f432..2aef8c0f 100644
--- a/logstash-output-elasticsearch.gemspec
+++ b/logstash-output-elasticsearch.gemspec
@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.4.1'
+  s.version         = '11.5.0'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"

From 944e5d707e0f19ec1e0b0edf662f328696af1dec Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 26 Jan 2022 09:31:30 +0100
Subject: [PATCH 19/30] revert precedence on TLSv1.3 being first

---
 lib/logstash/plugin_mixins/elasticsearch/api_configs.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
index 945cebf9..de075d10 100644
--- a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
+++ b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
@@ -66,7 +66,7 @@ module APIConfigs
         # Set the keystore password
         :keystore_password => { :validate => :password },
 
-        :ssl_supported_protocols => { :validate => :string, :default => ['TLSv1.2', 'TLSv1.3'], :list => true },
+        :ssl_supported_protocols => { :validate => :string, :default => ['TLSv1.3', 'TLSv1.2'], :list => true },
 
         # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
         # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use

From 3883483c4a9ab23f33442af081d806d4289d8cba Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 26 Jan 2022 09:31:47 +0100
Subject: [PATCH 20/30] Docs: the new configuration option

---
 docs/index.asciidoc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/index.asciidoc b/docs/index.asciidoc
index 85be8c8f..2993faad 100644
--- a/docs/index.asciidoc
+++ b/docs/index.asciidoc
@@ -355,6 +355,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-sniffing_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_supported_protocols>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
 | <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
@@ -1004,6 +1005,15 @@ Option to validate the server's certificate. Disabling this severely compromises
 For more information on disabling certificate verification please read
 https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
 
+[id="plugins-{type}s-{plugin}-ssl_supported_protocols"]
+===== `ssl_supported_protocols`
+
+  * Value type is <<string,string>>
+  * Default value is `['TLSv1.3', 'TLSv1.2']`
+
+List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
+The setting is a list of allowed protocol versions, for example `TLSv1.1`, `TLSv1.2`, `TLSv1.3`.
+
 [id="plugins-{type}s-{plugin}-template"]
 ===== `template`
 

From 76b87ebccdf92735d76391d1b1d15ac48be99ab0 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 26 Jan 2022 11:40:37 +0100
Subject: [PATCH 21/30] Test: curl -s -S

---
 spec/integration/outputs/index_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index b0f6bc82..17dc0119 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -62,7 +62,7 @@
 
   def curl_and_get_json_response(url, method: :get); require 'open3'
     begin
-      stdout, status = Open3.capture2("curl #{curl_opts} -X #{method.to_s.upcase} -k #{url}")
+      stdout, status = Open3.capture2("curl -s --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}")
     rescue Errno::ENOENT
       fail "curl not available, make sure curl binary is installed and available on $PATH"
     end

From f32f31906f98afeb3926dab21a7614d5774bd8d7 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Wed, 26 Jan 2022 15:09:13 +0100
Subject: [PATCH 22/30] Test: more curl-ing - detect http status code

---
 spec/integration/outputs/index_spec.rb | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index 17dc0119..649b8dbb 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -61,16 +61,24 @@
   let(:curl_opts) { nil }
 
   def curl_and_get_json_response(url, method: :get); require 'open3'
+    cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
     begin
-      stdout, status = Open3.capture2("curl -s --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}")
+      out, err, status = Open3.capture3(cmd)
     rescue Errno::ENOENT
       fail "curl not available, make sure curl binary is installed and available on $PATH"
     end
 
     if status.success?
-      LogStash::Json.load(stdout)
+      http_status = err.match(/< HTTP\/1.1 (.*?)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+      if http_status.strip[0].to_i > 2
+        warn out
+        fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+      end
+
+      LogStash::Json.load(out)
     else
-      fail "curl failed: #{status}\n  #{stdout}"
+      warn out
+      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
     end
   end
 

From 82d44878fa67c3d9ef82988f18afb337f4f89680 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Tue, 15 Mar 2022 13:07:50 +0100
Subject: [PATCH 23/30] only support TLSv1.1+ and validate enumeration

---
 lib/logstash/outputs/elasticsearch/http_client_builder.rb | 3 ++-
 lib/logstash/plugin_mixins/elasticsearch/api_configs.rb   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/logstash/outputs/elasticsearch/http_client_builder.rb b/lib/logstash/outputs/elasticsearch/http_client_builder.rb
index e575b566..ce59bae4 100644
--- a/lib/logstash/outputs/elasticsearch/http_client_builder.rb
+++ b/lib/logstash/outputs/elasticsearch/http_client_builder.rb
@@ -139,7 +139,8 @@ def self.setup_ssl(logger, params)
         ssl_options[:verify] = :disable # false accepts self-signed but still validates hostname
       end
 
-      ssl_options[:protocols] = params['ssl_supported_protocols'] if params['ssl_supported_protocols']
+      protocols = params['ssl_supported_protocols']
+      ssl_options[:protocols] = protocols if protocols && protocols.any?
 
       { ssl: ssl_options }
     end
diff --git a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
index de075d10..67bfeaad 100644
--- a/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
+++ b/lib/logstash/plugin_mixins/elasticsearch/api_configs.rb
@@ -66,7 +66,7 @@ module APIConfigs
         # Set the keystore password
         :keystore_password => { :validate => :password },
 
-        :ssl_supported_protocols => { :validate => :string, :default => ['TLSv1.3', 'TLSv1.2'], :list => true },
+        :ssl_supported_protocols => { :validate => ['TLSv1.1', 'TLSv1.2', 'TLSv1.3'], :default => [], :list => true },
 
         # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
         # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use

From 877377eda9d6c18b9b509ac9640604b2aacbe128 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Tue, 15 Mar 2022 13:26:57 +0100
Subject: [PATCH 24/30] adust docs

---
 docs/index.asciidoc | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/docs/index.asciidoc b/docs/index.asciidoc
index 48fb7a3b..4b507883 100644
--- a/docs/index.asciidoc
+++ b/docs/index.asciidoc
@@ -1009,10 +1009,12 @@ https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
 ===== `ssl_supported_protocols`
 
   * Value type is <<string,string>>
-  * Default value is `['TLSv1.3', 'TLSv1.2']`
+  * Allowed values are: `TLSv1.1`, `TLSv1.2`, 'TLSv1.3'
+  * There is no default value for this setting.
 
 List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
-The setting is a list of allowed protocol versions, for example `TLSv1.1`, `TLSv1.2`, `TLSv1.3`.
+The default depends on the JDK being used and with up-to-date Logstash/Java is usually `['TLSv1.2', 'TLSv1.3']`.
+When using Java 8 `TLSv1.3` is only supported since **8u262** (AdoptOpenJDK).
 
 [id="plugins-{type}s-{plugin}-template"]
 ===== `template`
@@ -1028,8 +1030,8 @@ If not set, the included template will be used.
 
   * Value type is <<string,string>>
   * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
-  ** ECS Compatibility disabled: `logstash`
-  ** ECS Compatibility enabled: `ecs-logstash`
+    ** ECS Compatibility disabled: `logstash`
+    ** ECS Compatibility enabled: `ecs-logstash`
 
 
 This configuration option defines how the template is named inside Elasticsearch.

From e7c6fb4dadd99cd27a25fbddc2c65681b688e449 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 28 Mar 2022 14:44:27 +0200
Subject: [PATCH 25/30] Docs: align docs with rest of plugins

---
 docs/index.asciidoc | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/docs/index.asciidoc b/docs/index.asciidoc
index 4b507883..0ad2beac 100644
--- a/docs/index.asciidoc
+++ b/docs/index.asciidoc
@@ -1009,12 +1009,18 @@ https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
 ===== `ssl_supported_protocols`
 
   * Value type is <<string,string>>
-  * Allowed values are: `TLSv1.1`, `TLSv1.2`, 'TLSv1.3'
-  * There is no default value for this setting.
+  * Allowed values are: `'TLSv1.1'`, `'TLSv1.2'`, `'TLSv1.3'`
+  * Default depends on the JDK being used. With up-to-date Logstash, the default is `['TLSv1.2', 'TLSv1.3']`.
+    `'TLSv1.1'` is not considered secure and is only provided for legacy applications.
 
 List of allowed SSL/TLS versions to use when establishing a connection to the Elasticsearch cluster.
-The default depends on the JDK being used and with up-to-date Logstash/Java is usually `['TLSv1.2', 'TLSv1.3']`.
-When using Java 8 `TLSv1.3` is only supported since **8u262** (AdoptOpenJDK).
+
+For Java 8 `'TLSv1.3'` is supported  only since **8u262** (AdoptOpenJDK), but requires that you set the
+`LS_JAVA_OPTS="-Djdk.tls.client.protocols=TLSv1.3"` system property in Logstash.
+
+NOTE: If you configure the plugin to use `'TLSv1.1'` on any recent JVM, such as the one packaged with Logstash,
+the protocol is disabled by default and needs to be enabled manually by changing `jdk.tls.disabledAlgorithms` in
+the *$JDK_HOME/conf/security/java.security* configuration file. That is, `TLSv1.1` needs to be removed from the list.
 
 [id="plugins-{type}s-{plugin}-template"]
 ===== `template`

From 0655b45b1219176a178631d83f84519014f0e8f5 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 28 Mar 2022 14:45:44 +0200
Subject: [PATCH 26/30] Temp: compare against 7.x

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 9eeaa6c8..ad06df96 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,6 +10,7 @@ env:
   - INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true LOG_LEVEL=info
   - INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info
+  - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_KEY_INVALID=true LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3 LOG_LEVEL=info

From d126939b2979f6c465c76dfd8b4de677efb03f47 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 28 Mar 2022 16:53:32 +0200
Subject: [PATCH 27/30] Test: a default super-user in ES

---
 .ci/Dockerfile.elasticsearch           | 1 +
 spec/integration/outputs/index_spec.rb | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/.ci/Dockerfile.elasticsearch b/.ci/Dockerfile.elasticsearch
index 8aab4d4c..5ad0cec1 100644
--- a/.ci/Dockerfile.elasticsearch
+++ b/.ci/Dockerfile.elasticsearch
@@ -30,5 +30,6 @@ RUN if [ "$SECURE_INTEGRATION" = "true" ] && [ ! -z "$ES_SSL_SUPPORTED_PROTOCOLS
 
 RUN cat $es_yml
 
+RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd admin -p elastic -r superuser; fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser; fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser; fi
diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index 649b8dbb..c645f80f 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -60,6 +60,9 @@
 
   let(:curl_opts) { nil }
 
+  let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
+  let(:es_admin_pass) { 'elastic' }
+
   def curl_and_get_json_response(url, method: :get); require 'open3'
     cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
     begin

From 09e43010ebffdfb74d37138d8413a9a42b5ca14c Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 28 Mar 2022 16:54:01 +0200
Subject: [PATCH 28/30] Test: proper status code detection from curl

---
 spec/integration/outputs/index_spec.rb | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index c645f80f..c27c2fab 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -72,10 +72,16 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
     end
 
     if status.success?
-      http_status = err.match(/< HTTP\/1.1 (.*?)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+      http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+
       if http_status.strip[0].to_i > 2
-        warn out
-        fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+        error = (LogStash::Json.load(out)['error']) rescue nil
+        if error
+          fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}"
+        else
+          warn out
+          fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+        end
       end
 
       LogStash::Json.load(out)

From 37912d3d18e8304d9c9aa8f5e5d1bc7a6c4ae7c4 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 28 Mar 2022 16:54:26 +0200
Subject: [PATCH 29/30] Test: always require auth for _refresh etc (due 8.2)

---
 spec/integration/outputs/index_spec.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb
index c27c2fab..21b46dab 100644
--- a/spec/integration/outputs/index_spec.rb
+++ b/spec/integration/outputs/index_spec.rb
@@ -220,7 +220,7 @@ def curl_and_get_json_response(url, method: :get); require 'open3'
 
     else
 
-      let(:curl_opts) { "#{super()} --tlsv1.2 --tls-max 1.3" }
+      let(:curl_opts) { "#{super()} --tlsv1.2 --tls-max 1.3 -u #{es_admin}:#{es_admin_pass}" } # due ES 8.x we need user/password
 
       it_behaves_like("an indexer", true)
 

From 8ae8452fd87caf9a3cb0c91710b6678f4970c628 Mon Sep 17 00:00:00 2001
From: kares <self@kares.org>
Date: Mon, 28 Mar 2022 16:54:45 +0200
Subject: [PATCH 30/30] Revert "Temp: compare against 7.x"

This reverts commit 0655b45b1219176a178631d83f84519014f0e8f5.
---
 .travis.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index ad06df96..9eeaa6c8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,6 @@ env:
   - INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true LOG_LEVEL=info
   - INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info
-  - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_KEY_INVALID=true LOG_LEVEL=info
   - SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3 LOG_LEVEL=info