From d3366376d8b71b614a709743e5748e401509b18d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 20 Mar 2018 15:27:07 -0400 Subject: [PATCH 01/27] Init --- .../client/RestHighLevelClient.java | 10 +- .../CustomRestHighLevelClientTests.java | 6 +- .../client/RestHighLevelClientExtTests.java | 2 +- .../client/RestHighLevelClientTests.java | 17 +- .../client/AbstractRestClientActions.java | 325 ++++++++++++++ .../java/org/elasticsearch/client/Node.java | 78 ++++ .../elasticsearch/client/NodeSelector.java | 49 +++ .../org/elasticsearch/client/RestClient.java | 403 ++++-------------- .../client/RestClientActions.java | 181 ++++++++ .../elasticsearch/client/RestClientView.java | 63 +++ .../RestClientDocumentation.java | 12 + .../sniff/ElasticsearchHostsSniffer.java | 2 +- 12 files changed, 815 insertions(+), 333 deletions(-) create mode 100644 client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java create mode 100644 client/rest/src/main/java/org/elasticsearch/client/Node.java create mode 100644 client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java create mode 100644 client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java create mode 100644 client/rest/src/main/java/org/elasticsearch/client/RestClientView.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index bf80aa7720741..85d79f2af6ccf 100755 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -181,9 +181,9 @@ */ public class RestHighLevelClient implements Closeable { - private final RestClient client; + private final RestClientActions client; private final NamedXContentRegistry registry; - private final CheckedConsumer doClose; + private final CheckedConsumer doClose; private final IndicesClient indicesClient = new IndicesClient(this); private final ClusterClient clusterClient = new ClusterClient(this); @@ -201,7 +201,7 @@ public RestHighLevelClient(RestClientBuilder restClientBuilder) { * {@link RestClient} to be used to perform requests and parsers for custom response sections added to Elasticsearch through plugins. */ protected RestHighLevelClient(RestClientBuilder restClientBuilder, List namedXContentEntries) { - this(restClientBuilder.build(), RestClient::close, namedXContentEntries); + this(restClientBuilder.build(), (RestClientActions c) -> ((RestClient) c).close(), namedXContentEntries); } /** @@ -211,7 +211,7 @@ protected RestHighLevelClient(RestClientBuilder restClientBuilder, List doClose, + protected RestHighLevelClient(RestClientActions restClient, CheckedConsumer doClose, List namedXContentEntries) { this.client = Objects.requireNonNull(restClient, "restClient must not be null"); this.doClose = Objects.requireNonNull(doClose, "doClose consumer must not be null"); @@ -223,7 +223,7 @@ protected RestHighLevelClient(RestClient restClient, CheckedConsumer mockPerformRequest((Header) mock.getArguments()[4])) @@ -172,8 +172,8 @@ private Response mockPerformRequest(Header httpHeader) throws IOException { */ static class CustomRestClient extends RestHighLevelClient { - private CustomRestClient(RestClient restClient) { - super(restClient, RestClient::close, Collections.emptyList()); + private CustomRestClient(RestClientActions restClient) { + super(restClient, c -> {}, Collections.emptyList()); } MainResponse custom(MainRequest mainRequest, Header... headers) throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java index 970ffb15f083b..7903eb0a6a3ba 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java @@ -69,7 +69,7 @@ public void testParseEntityCustomResponseSection() throws IOException { private static class RestHighLevelClientExt extends RestHighLevelClient { private RestHighLevelClientExt(RestClient restClient) { - super(restClient, RestClient::close, getNamedXContentsExt()); + super(restClient, c -> restClient.close(), getNamedXContentsExt()); } private static List getNamedXContentsExt() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index b8315bd59fa43..184833c00452a 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -54,6 +54,7 @@ import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -112,22 +113,26 @@ public class RestHighLevelClientTests extends ESTestCase { private static final ProtocolVersion HTTP_PROTOCOL = new ProtocolVersion("http", 1, 1); private static final RequestLine REQUEST_LINE = new BasicRequestLine(HttpGet.METHOD_NAME, "/", HTTP_PROTOCOL); - private RestClient restClient; + private RestClientActions restClient; + private CheckedConsumer doClose; private RestHighLevelClient restHighLevelClient; @Before public void initClient() { - restClient = mock(RestClient.class); - restHighLevelClient = new RestHighLevelClient(restClient, RestClient::close, Collections.emptyList()); + restClient = mock(RestClientActions.class); + @SuppressWarnings("unchecked") + CheckedConsumer doClose = mock(CheckedConsumer.class); + this.doClose = doClose; + restHighLevelClient = new RestHighLevelClient(restClient, doClose, Collections.emptyList()); } public void testCloseIsIdempotent() throws IOException { restHighLevelClient.close(); - verify(restClient, times(1)).close(); + verify(doClose, times(1)).accept(restClient); restHighLevelClient.close(); - verify(restClient, times(2)).close(); + verify(doClose, times(2)).accept(restClient); restHighLevelClient.close(); - verify(restClient, times(3)).close(); + verify(doClose, times(3)).accept(restClient); } public void testPingSuccessful() throws IOException { diff --git a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java new file mode 100644 index 0000000000000..52ef58f011e34 --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java @@ -0,0 +1,325 @@ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import javax.net.ssl.SSLHandshakeException; + +import org.apache.http.ConnectionClosedException; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.conn.ConnectTimeoutException; + +abstract class AbstractRestClientActions implements RestClientActions { + protected abstract void performRequestAsyncNoCatch(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + ResponseListener responseListener, Header[] headers); + + protected abstract SyncResponseListener syncResponseListener(); + + /** + * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response + * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without parameters + * and request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param headers the optional request headers + * @return the response returned by Elasticsearch + * @throws IOException in case of a problem or the connection was aborted + * @throws ClientProtocolException in case of an http protocol error + * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error + */ + @Override + public final Response performRequest(String method, String endpoint, Header... headers) throws IOException { + return performRequest(method, endpoint, Collections.emptyMap(), null, headers); + } + + /** + * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response + * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param headers the optional request headers + * @return the response returned by Elasticsearch + * @throws IOException in case of a problem or the connection was aborted + * @throws ClientProtocolException in case of an http protocol error + * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error + */ + @Override + public final Response performRequest(String method, String endpoint, Map params, Header... headers) throws IOException { + return performRequest(method, endpoint, params, (HttpEntity)null, headers); + } + + /** + * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response + * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, Header...)} + * which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance, + * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param headers the optional request headers + * @return the response returned by Elasticsearch + * @throws IOException in case of a problem or the connection was aborted + * @throws ClientProtocolException in case of an http protocol error + * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error + */ + @Override + public final Response performRequest(String method, String endpoint, Map params, + HttpEntity entity, Header... headers) throws IOException { + return performRequest(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, headers); + } + + /** + * Sends a request to the Elasticsearch cluster that the client points to. Blocks until the request is completed and returns + * its response or fails by throwing an exception. Selects a host out of the provided ones in a round-robin fashion. Failing hosts + * are marked dead and retried after a certain amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times + * they previously failed (the more failures, the later they will be retried). In case of failures all of the alive nodes (or dead + * nodes that deserve a retry) are retried until one responds or none of them does, in which case an {@link IOException} will be thrown. + * + * This method works by performing an asynchronous call and waiting + * for the result. If the asynchronous call throws an exception we wrap + * it and rethrow it so that the stack trace attached to the exception + * contains the call site. While we attempt to preserve the original + * exception this isn't always possible and likely haven't covered all of + * the cases. You can get the original exception from + * {@link Exception#getCause()}. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one + * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP + * connection on the client side. + * @param headers the optional request headers + * @return the response returned by Elasticsearch + * @throws IOException in case of a problem or the connection was aborted + * @throws ClientProtocolException in case of an http protocol error + * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error + */ + @Override + public final Response performRequest(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + Header... headers) throws IOException { + SyncResponseListener listener = syncResponseListener(); + performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, + listener, headers); + return listener.get(); + } + + /** + * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead + * the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to + * {@link #performRequestAsync(String, String, Map, HttpEntity, ResponseListener, Header...)} but without parameters and request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + public final void performRequestAsync(String method, String endpoint, ResponseListener responseListener, Header... headers) { + performRequestAsync(method, endpoint, Collections.emptyMap(), null, responseListener, headers); + } + + /** + * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead + * the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to + * {@link #performRequestAsync(String, String, Map, HttpEntity, ResponseListener, Header...)} but without request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + @Override + public final void performRequestAsync(String method, String endpoint, Map params, + ResponseListener responseListener, Header... headers) { + performRequestAsync(method, endpoint, params, null, responseListener, headers); + } + + /** + * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead + * the provided {@link ResponseListener} will be notified upon completion or failure. + * Shortcut to {@link #performRequestAsync(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, ResponseListener, + * Header...)} which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance, + * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + @Override + public final void performRequestAsync(String method, String endpoint, Map params, + HttpEntity entity, ResponseListener responseListener, Header... headers) { + performRequestAsync(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, responseListener, headers); + } + + /** + * Sends a request to the Elasticsearch cluster that the client points to. The request is executed asynchronously + * and the provided {@link ResponseListener} gets notified upon request completion or failure. + * Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain + * amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously failed (the more failures, + * the later they will be retried). In case of failures all of the alive nodes (or dead nodes that deserve a retry) are retried + * until one responds or none of them does, in which case an {@link IOException} will be thrown. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one + * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP + * connection on the client side. + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + @Override + public final void performRequestAsync(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + ResponseListener responseListener, Header... headers) { + try { + performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, + responseListener, headers); + } catch (Exception e) { + responseListener.onFailure(e); + } + } + + /** + * Listener used in any sync performRequest calls, it waits for a response or an exception back up to a timeout + */ + static class SyncResponseListener implements ResponseListener { + private final CountDownLatch latch = new CountDownLatch(1); + private final AtomicReference response = new AtomicReference<>(); + private final AtomicReference exception = new AtomicReference<>(); + + private final long timeout; + + SyncResponseListener(long timeout) { + assert timeout > 0; + this.timeout = timeout; + } + + @Override + public void onSuccess(Response response) { + Objects.requireNonNull(response, "response must not be null"); + boolean wasResponseNull = this.response.compareAndSet(null, response); + if (wasResponseNull == false) { + throw new IllegalStateException("response is already set"); + } + + latch.countDown(); + } + + @Override + public void onFailure(Exception exception) { + Objects.requireNonNull(exception, "exception must not be null"); + boolean wasExceptionNull = this.exception.compareAndSet(null, exception); + if (wasExceptionNull == false) { + throw new IllegalStateException("exception is already set"); + } + latch.countDown(); + } + + /** + * Waits (up to a timeout) for some result of the request: either a response, or an exception. + */ + Response get() throws IOException { + try { + //providing timeout is just a safety measure to prevent everlasting waits + //the different client timeouts should already do their jobs + if (latch.await(timeout, TimeUnit.MILLISECONDS) == false) { + throw new IOException("listener timeout after waiting for [" + timeout + "] ms"); + } + } catch (InterruptedException e) { + throw new RuntimeException("thread waiting for the response was interrupted", e); + } + + Exception exception = this.exception.get(); + Response response = this.response.get(); + if (exception != null) { + if (response != null) { + IllegalStateException e = new IllegalStateException("response and exception are unexpectedly set at the same time"); + e.addSuppressed(exception); + throw e; + } + /* + * Wrap and rethrow whatever exception we received, copying the type + * where possible so the synchronous API looks as much as possible + * like the asynchronous API. We wrap the exception so that the caller's + * signature shows up in any exception we throw. + */ + if (exception instanceof ResponseException) { + throw new ResponseException((ResponseException) exception); + } + if (exception instanceof ConnectTimeoutException) { + ConnectTimeoutException e = new ConnectTimeoutException(exception.getMessage()); + e.initCause(exception); + throw e; + } + if (exception instanceof SocketTimeoutException) { + SocketTimeoutException e = new SocketTimeoutException(exception.getMessage()); + e.initCause(exception); + throw e; + } + if (exception instanceof ConnectionClosedException) { + ConnectionClosedException e = new ConnectionClosedException(exception.getMessage()); + e.initCause(exception); + throw e; + } + if (exception instanceof SSLHandshakeException) { + SSLHandshakeException e = new SSLHandshakeException(exception.getMessage()); + e.initCause(exception); + throw e; + } + if (exception instanceof IOException) { + throw new IOException(exception.getMessage(), exception); + } + if (exception instanceof RuntimeException){ + throw new RuntimeException(exception.getMessage(), exception); + } + throw new RuntimeException("error while performing request", exception); + } + + if (response == null) { + throw new IllegalStateException("response not set and no exception caught either"); + } + return response; + } + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/Node.java b/client/rest/src/main/java/org/elasticsearch/client/Node.java new file mode 100644 index 0000000000000..84e8ec1e310d4 --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/Node.java @@ -0,0 +1,78 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.util.Objects; + +import org.apache.http.HttpHost; + +public final class Node { + interface Roles { + boolean master(); + boolean data(); + boolean ingest(); + } + + public static Node build(HttpHost host) { + return build(host, null); + } + + public static Node build(HttpHost host, Roles roles) { + return new Node(host, roles); + } + + private final HttpHost host; + private final Roles roles; + + private Node(HttpHost host, Roles roles) { + this.host = Objects.requireNonNull(host, "host cannot be null"); + this.roles = roles; + } + + public HttpHost host() { + return host; + } + + public Roles roles() { + return roles; + } + + @Override + public String toString() { + return host.toString(); + } + + @Override + public boolean equals(Object obj) { + if (obj == this) { + return true; + } + if (obj == null || obj.getClass() != getClass()) { + return false; + } + Node other = (Node) obj; + return host.equals(other.host); + } + + @Override + public int hashCode() { + return host.hashCode(); + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java new file mode 100644 index 0000000000000..04bd70e1aeca4 --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java @@ -0,0 +1,49 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +public interface NodeSelector { + /** + * Selector that matches any node. + */ + NodeSelector ANY = new NodeSelector() { + @Override + public boolean select(Node node) { + return true; + } + }; + + /** + * Selector that matches any node that doesn't have the master roles + * or doesn't have any information about roles. + */ + NodeSelector NOT_MASTER = new NodeSelector() { + @Override + public boolean select(Node node) { + return node.roles() == null || false == node.roles().master(); + } + }; + + /** + * Return {@code true} if this node should be used for requests, {@code false} + * otherwise. + */ + boolean select(Node node); +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 4aa1a9d815cf4..d30d200791770 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -16,18 +16,17 @@ * specific language governing permissions and limitations * under the License. */ + package org.elasticsearch.client; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.ConnectionClosedException; import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpResponse; import org.apache.http.client.AuthCache; -import org.apache.http.client.ClientProtocolException; import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpOptions; @@ -39,7 +38,6 @@ import org.apache.http.client.protocol.HttpClientContext; import org.apache.http.client.utils.URIBuilder; import org.apache.http.concurrent.FutureCallback; -import org.apache.http.conn.ConnectTimeoutException; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.BasicAuthCache; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; @@ -49,7 +47,6 @@ import java.io.Closeable; import java.io.IOException; -import java.net.SocketTimeoutException; import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; @@ -67,12 +64,9 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import javax.net.ssl.SSLHandshakeException; /** * Client that connects to an Elasticsearch cluster through HTTP. @@ -91,7 +85,7 @@ *

* Requests can be traced by enabling trace logging for "tracer". The trace logger outputs requests and responses in curl format. */ -public class RestClient implements Closeable { +public class RestClient extends AbstractRestClientActions implements Closeable { private static final Log logger = LogFactory.getLog(RestClient.class); @@ -102,8 +96,8 @@ public class RestClient implements Closeable { private final long maxRetryTimeoutMillis; private final String pathPrefix; private final AtomicInteger lastHostIndex = new AtomicInteger(0); - private volatile HostTuple> hostTuple; - private final ConcurrentMap blacklist = new ConcurrentHashMap<>(); + private volatile HostTuple> hostTuple; + private final ConcurrentMap blacklist = new ConcurrentHashMap<>(); private final FailureListener failureListener; RestClient(CloseableHttpAsyncClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders, @@ -128,194 +122,56 @@ public static RestClientBuilder builder(HttpHost... hosts) { * Replaces the hosts that the client communicates with. * @see HttpHost */ - public synchronized void setHosts(HttpHost... hosts) { + public void setHosts(HttpHost... hosts) { if (hosts == null || hosts.length == 0) { throw new IllegalArgumentException("hosts must not be null nor empty"); } - Set httpHosts = new HashSet<>(); - AuthCache authCache = new BasicAuthCache(); - for (HttpHost host : hosts) { - Objects.requireNonNull(host, "host cannot be null"); - httpHosts.add(host); - authCache.put(host, new BasicScheme()); + Node[] nodes = new Node[hosts.length]; + for (int i = 0; i < hosts.length; i++) { + final HttpHost host = hosts[i]; + nodes[i] = Node.build(host); } - this.hostTuple = new HostTuple<>(Collections.unmodifiableSet(httpHosts), authCache); - this.blacklist.clear(); - } - - /** - * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response - * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without parameters - * and request body. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param headers the optional request headers - * @return the response returned by Elasticsearch - * @throws IOException in case of a problem or the connection was aborted - * @throws ClientProtocolException in case of an http protocol error - * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error - */ - public Response performRequest(String method, String endpoint, Header... headers) throws IOException { - return performRequest(method, endpoint, Collections.emptyMap(), null, headers); + setNodes(nodes); } - /** - * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response - * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without request body. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param headers the optional request headers - * @return the response returned by Elasticsearch - * @throws IOException in case of a problem or the connection was aborted - * @throws ClientProtocolException in case of an http protocol error - * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error - */ - public Response performRequest(String method, String endpoint, Map params, Header... headers) throws IOException { - return performRequest(method, endpoint, params, (HttpEntity)null, headers); + public synchronized void setNodes(Node... nodes) { + if (nodes == null || nodes.length == 0) { + throw new IllegalArgumentException("nodes must not be null nor empty"); + } + Set newNodes = new HashSet<>(); + AuthCache authCache = new BasicAuthCache(); + for (Node node : nodes) { + Objects.requireNonNull(node, "node cannot be null"); + newNodes.add(node); + authCache.put(node.host(), new BasicScheme()); + } + this.hostTuple = new HostTuple<>(Collections.unmodifiableSet(newNodes), authCache); + this.blacklist.clear(); } - /** - * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response - * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, Header...)} - * which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance, - * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param entity the body of the request, null if not applicable - * @param headers the optional request headers - * @return the response returned by Elasticsearch - * @throws IOException in case of a problem or the connection was aborted - * @throws ClientProtocolException in case of an http protocol error - * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error - */ - public Response performRequest(String method, String endpoint, Map params, - HttpEntity entity, Header... headers) throws IOException { - return performRequest(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, headers); + @Override // NOCOMMIT this shouldn't be public + public SyncResponseListener syncResponseListener() { + return new SyncResponseListener(maxRetryTimeoutMillis); } - /** - * Sends a request to the Elasticsearch cluster that the client points to. Blocks until the request is completed and returns - * its response or fails by throwing an exception. Selects a host out of the provided ones in a round-robin fashion. Failing hosts - * are marked dead and retried after a certain amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times - * they previously failed (the more failures, the later they will be retried). In case of failures all of the alive nodes (or dead - * nodes that deserve a retry) are retried until one responds or none of them does, in which case an {@link IOException} will be thrown. - * - * This method works by performing an asynchronous call and waiting - * for the result. If the asynchronous call throws an exception we wrap - * it and rethrow it so that the stack trace attached to the exception - * contains the call site. While we attempt to preserve the original - * exception this isn't always possible and likely haven't covered all of - * the cases. You can get the original exception from - * {@link Exception#getCause()}. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param entity the body of the request, null if not applicable - * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one - * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP - * connection on the client side. - * @param headers the optional request headers - * @return the response returned by Elasticsearch - * @throws IOException in case of a problem or the connection was aborted - * @throws ClientProtocolException in case of an http protocol error - * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error - */ - public Response performRequest(String method, String endpoint, Map params, - HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - Header... headers) throws IOException { - SyncResponseListener listener = new SyncResponseListener(maxRetryTimeoutMillis); + @Override + protected void performRequestAsyncNoCatch(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + ResponseListener responseListener, Header[] headers) { + // Requests made directly to the client use the noop NodeSelector. + NodeSelector nodeSelector = NodeSelector.ANY; performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, - listener, headers); - return listener.get(); - } - - /** - * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead - * the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to - * {@link #performRequestAsync(String, String, Map, HttpEntity, ResponseListener, Header...)} but without parameters and request body. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails - * @param headers the optional request headers - */ - public void performRequestAsync(String method, String endpoint, ResponseListener responseListener, Header... headers) { - performRequestAsync(method, endpoint, Collections.emptyMap(), null, responseListener, headers); - } - - /** - * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead - * the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to - * {@link #performRequestAsync(String, String, Map, HttpEntity, ResponseListener, Header...)} but without request body. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails - * @param headers the optional request headers - */ - public void performRequestAsync(String method, String endpoint, Map params, - ResponseListener responseListener, Header... headers) { - performRequestAsync(method, endpoint, params, null, responseListener, headers); + responseListener, nodeSelector, headers); } - /** - * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead - * the provided {@link ResponseListener} will be notified upon completion or failure. - * Shortcut to {@link #performRequestAsync(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, ResponseListener, - * Header...)} which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance, - * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param entity the body of the request, null if not applicable - * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails - * @param headers the optional request headers - */ - public void performRequestAsync(String method, String endpoint, Map params, - HttpEntity entity, ResponseListener responseListener, Header... headers) { - performRequestAsync(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, responseListener, headers); - } - - /** - * Sends a request to the Elasticsearch cluster that the client points to. The request is executed asynchronously - * and the provided {@link ResponseListener} gets notified upon request completion or failure. - * Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain - * amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously failed (the more failures, - * the later they will be retried). In case of failures all of the alive nodes (or dead nodes that deserve a retry) are retried - * until one responds or none of them does, in which case an {@link IOException} will be thrown. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param entity the body of the request, null if not applicable - * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one - * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP - * connection on the client side. - * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails - * @param headers the optional request headers - */ - public void performRequestAsync(String method, String endpoint, Map params, - HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, Header... headers) { - try { - performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, - responseListener, headers); - } catch (Exception e) { - responseListener.onFailure(e); - } + @Override + public RestClientActions withNodeSelector(NodeSelector nodeSelector) { + return new RestClientView(this, nodeSelector); } void performRequestAsyncNoCatch(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, Header... headers) { + ResponseListener responseListener, NodeSelector nodeSelector, Header[] headers) { Objects.requireNonNull(params, "params must not be null"); Map requestParams = new HashMap<>(params); //ignore is a special parameter supported by the clients, shouldn't be sent to es @@ -348,17 +204,17 @@ void performRequestAsyncNoCatch(String method, String endpoint, Map> hostTuple, final HttpRequestBase request, + private void performRequestAsync(final long startTime, final HostTuple> hostTuple, final HttpRequestBase request, final Set ignoreErrorCodes, final HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, final FailureTrackingResponseListener listener) { - final HttpHost host = hostTuple.hosts.next(); + final Node node = hostTuple.hosts.next(); //we stream the request body if the entity allows for it - final HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request); + final HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(node.host(), request); final HttpAsyncResponseConsumer asyncResponseConsumer = httpAsyncResponseConsumerFactory.createHttpAsyncResponseConsumer(); final HttpClientContext context = HttpClientContext.create(); @@ -367,21 +223,21 @@ private void performRequestAsync(final long startTime, final HostTuple> nextHost() { - final HostTuple> hostTuple = this.hostTuple; - Collection nextHosts = Collections.emptySet(); + private HostTuple> nextHost(NodeSelector nodeSelector) { + final HostTuple> hostTuple = this.hostTuple; + Collection nextHosts = Collections.emptySet(); do { - Set filteredHosts = new HashSet<>(hostTuple.hosts); - for (Map.Entry entry : blacklist.entrySet()) { + Set filteredHosts = new HashSet<>(hostTuple.hosts); + for (Iterator hostItr = filteredHosts.iterator(); hostItr.hasNext();) { + final Node node = hostItr.next(); + if (false == nodeSelector.select(node)) { + hostItr.remove(); + } + } + for (Map.Entry entry : blacklist.entrySet()) { if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) { filteredHosts.remove(entry.getKey()); } } if (filteredHosts.isEmpty()) { - //last resort: if there are no good host to use, return a single dead one, the one that's closest to being retried - List> sortedHosts = new ArrayList<>(blacklist.entrySet()); + /* + * Last resort: If there are no good host to use, return a single dead one, + * the one that's closest to being retried *and* matches the selector. + */ + List> sortedHosts = new ArrayList<>(blacklist.entrySet()); if (sortedHosts.size() > 0) { - Collections.sort(sortedHosts, new Comparator>() { + Collections.sort(sortedHosts, new Comparator>() { @Override - public int compare(Map.Entry o1, Map.Entry o2) { + public int compare(Map.Entry o1, Map.Entry o2) { return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos()); } }); - HttpHost deadHost = sortedHosts.get(0).getKey(); - logger.trace("resurrecting host [" + deadHost + "]"); - nextHosts = Collections.singleton(deadHost); + Iterator> nodeItr = sortedHosts.iterator(); + while (nodeItr.hasNext()) { + final Node deadNode = nodeItr.next().getKey(); + if (nodeSelector.select(deadNode)) { + logger.trace("resurrecting host [" + deadNode.host() + "]"); + nextHosts = Collections.singleton(deadNode); + break; + } + } } + // NOCOMMIT we get here if the selector rejects all hosts } else { - List rotatedHosts = new ArrayList<>(filteredHosts); + List rotatedHosts = new ArrayList<>(filteredHosts); Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement()); nextHosts = rotatedHosts; } @@ -488,10 +360,10 @@ public int compare(Map.Entry o1, Map.Entry response = new AtomicReference<>(); - private final AtomicReference exception = new AtomicReference<>(); - - private final long timeout; - - SyncResponseListener(long timeout) { - assert timeout > 0; - this.timeout = timeout; - } - - @Override - public void onSuccess(Response response) { - Objects.requireNonNull(response, "response must not be null"); - boolean wasResponseNull = this.response.compareAndSet(null, response); - if (wasResponseNull == false) { - throw new IllegalStateException("response is already set"); - } - - latch.countDown(); - } - - @Override - public void onFailure(Exception exception) { - Objects.requireNonNull(exception, "exception must not be null"); - boolean wasExceptionNull = this.exception.compareAndSet(null, exception); - if (wasExceptionNull == false) { - throw new IllegalStateException("exception is already set"); - } - latch.countDown(); - } - - /** - * Waits (up to a timeout) for some result of the request: either a response, or an exception. - */ - Response get() throws IOException { - try { - //providing timeout is just a safety measure to prevent everlasting waits - //the different client timeouts should already do their jobs - if (latch.await(timeout, TimeUnit.MILLISECONDS) == false) { - throw new IOException("listener timeout after waiting for [" + timeout + "] ms"); - } - } catch (InterruptedException e) { - throw new RuntimeException("thread waiting for the response was interrupted", e); - } - - Exception exception = this.exception.get(); - Response response = this.response.get(); - if (exception != null) { - if (response != null) { - IllegalStateException e = new IllegalStateException("response and exception are unexpectedly set at the same time"); - e.addSuppressed(exception); - throw e; - } - /* - * Wrap and rethrow whatever exception we received, copying the type - * where possible so the synchronous API looks as much as possible - * like the asynchronous API. We wrap the exception so that the caller's - * signature shows up in any exception we throw. - */ - if (exception instanceof ResponseException) { - throw new ResponseException((ResponseException) exception); - } - if (exception instanceof ConnectTimeoutException) { - ConnectTimeoutException e = new ConnectTimeoutException(exception.getMessage()); - e.initCause(exception); - throw e; - } - if (exception instanceof SocketTimeoutException) { - SocketTimeoutException e = new SocketTimeoutException(exception.getMessage()); - e.initCause(exception); - throw e; - } - if (exception instanceof ConnectionClosedException) { - ConnectionClosedException e = new ConnectionClosedException(exception.getMessage()); - e.initCause(exception); - throw e; - } - if (exception instanceof SSLHandshakeException) { - SSLHandshakeException e = new SSLHandshakeException(exception.getMessage()); - e.initCause(exception); - throw e; - } - if (exception instanceof IOException) { - throw new IOException(exception.getMessage(), exception); - } - if (exception instanceof RuntimeException){ - throw new RuntimeException(exception.getMessage(), exception); - } - throw new RuntimeException("error while performing request", exception); - } - - if (response == null) { - throw new IllegalStateException("response not set and no exception caught either"); - } - return response; - } - } - /** * Listener that allows to be notified whenever a failure happens. Useful when sniffing is enabled, so that we can sniff on failure. * The default implementation is a no-op. diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java new file mode 100644 index 0000000000000..be9aaaa05eefb --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java @@ -0,0 +1,181 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.io.IOException; +import java.util.Map; + +import org.apache.http.Header; +import org.apache.http.HttpEntity; + +public interface RestClientActions { + /** + * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response + * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without parameters + * and request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param headers the optional request headers + * @return the response returned by Elasticsearch + * @throws IOException in case of a problem or the connection was aborted + * @throws ClientProtocolException in case of an http protocol error + * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error + */ + Response performRequest(String method, String endpoint, Header... headers) throws IOException; + + /** + * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response + * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param headers the optional request headers + * @return the response returned by Elasticsearch + * @throws IOException in case of a problem or the connection was aborted + * @throws ClientProtocolException in case of an http protocol error + * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error + */ + Response performRequest(String method, String endpoint, Map params, Header... headers) throws IOException; + + /** + * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response + * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, Header...)} + * which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance, + * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param headers the optional request headers + * @return the response returned by Elasticsearch + * @throws IOException in case of a problem or the connection was aborted + * @throws ClientProtocolException in case of an http protocol error + * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error + */ + Response performRequest(String method, String endpoint, Map params, + HttpEntity entity, Header... headers) throws IOException; + + /** + * Sends a request to the Elasticsearch cluster that the client points to. Blocks until the request is completed and returns + * its response or fails by throwing an exception. Selects a host out of the provided ones in a round-robin fashion. Failing hosts + * are marked dead and retried after a certain amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times + * they previously failed (the more failures, the later they will be retried). In case of failures all of the alive nodes (or dead + * nodes that deserve a retry) are retried until one responds or none of them does, in which case an {@link IOException} will be thrown. + * + * This method works by performing an asynchronous call and waiting + * for the result. If the asynchronous call throws an exception we wrap + * it and rethrow it so that the stack trace attached to the exception + * contains the call site. While we attempt to preserve the original + * exception this isn't always possible and likely haven't covered all of + * the cases. You can get the original exception from + * {@link Exception#getCause()}. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one + * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP + * connection on the client side. + * @param headers the optional request headers + * @return the response returned by Elasticsearch + * @throws IOException in case of a problem or the connection was aborted + * @throws ClientProtocolException in case of an http protocol error + * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error + */ + Response performRequest(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + Header... headers) throws IOException; + + /** + * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead + * the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to + * {@link #performRequestAsync(String, String, Map, HttpEntity, ResponseListener, Header...)} but without parameters and request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + void performRequestAsync(String method, String endpoint, ResponseListener responseListener, Header... headers); + + /** + * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead + * the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to + * {@link #performRequestAsync(String, String, Map, HttpEntity, ResponseListener, Header...)} but without request body. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + void performRequestAsync(String method, String endpoint, Map params, + ResponseListener responseListener, Header... headers); + + /** + * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead + * the provided {@link ResponseListener} will be notified upon completion or failure. + * Shortcut to {@link #performRequestAsync(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, ResponseListener, + * Header...)} which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance, + * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + void performRequestAsync(String method, String endpoint, Map params, + HttpEntity entity, ResponseListener responseListener, Header... headers); + + /** + * Sends a request to the Elasticsearch cluster that the client points to. The request is executed asynchronously + * and the provided {@link ResponseListener} gets notified upon request completion or failure. + * Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain + * amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously failed (the more failures, + * the later they will be retried). In case of failures all of the alive nodes (or dead nodes that deserve a retry) are retried + * until one responds or none of them does, in which case an {@link IOException} will be thrown. + * + * @param method the http method + * @param endpoint the path of the request (without host and port) + * @param params the query_string parameters + * @param entity the body of the request, null if not applicable + * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one + * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP + * connection on the client side. + * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails + * @param headers the optional request headers + */ + void performRequestAsync(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + ResponseListener responseListener, Header... headers); + + /** + * Create a view that only runs requests on selected nodes. This object + * has no state of its own and backs everything to the {@link RestClient} + * that created it. + */ + RestClientActions withNodeSelector(NodeSelector nodeSelector); +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java new file mode 100644 index 0000000000000..be246d8f31f3a --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.util.Map; + +import org.apache.http.Header; +import org.apache.http.HttpEntity; + +/** + * Light weight view into a {@link RestClient} that doesn't have any state of its own. + */ +class RestClientView extends AbstractRestClientActions { + private final RestClient delegate; + private final NodeSelector nodeSelector; + + protected RestClientView(RestClient delegate, NodeSelector nodeSelector) { + this.delegate = delegate; + this.nodeSelector = nodeSelector; + } + + @Override + protected final void performRequestAsyncNoCatch(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + ResponseListener responseListener, Header[] headers) { + delegate.performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, + responseListener, nodeSelector, headers); + } + + @Override + protected final SyncResponseListener syncResponseListener() { + return delegate.syncResponseListener(); + } + + @Override + public final RestClientView withNodeSelector(final NodeSelector nodeSelector) { + final NodeSelector inner = this.nodeSelector; + NodeSelector combo = new NodeSelector() { + @Override + public boolean select(Node node) { + return inner.select(node) && nodeSelector.select(node); + } + }; + return new RestClientView(delegate, combo); + } +} diff --git a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java index 1bad6b5f6d6fd..085ed06e43935 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java +++ b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java @@ -28,6 +28,7 @@ import org.apache.http.client.CredentialsProvider; import org.apache.http.client.config.RequestConfig; import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.impl.nio.reactor.IOReactorConfig; @@ -37,9 +38,11 @@ import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseListener; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientActions; import org.elasticsearch.client.RestClientBuilder; import javax.net.ssl.SSLContext; @@ -253,7 +256,16 @@ public void onFailure(Exception exception) { latch.await(); //end::rest-client-async-example } + } + public void testNodeSelector() throws IOException { + try (RestClient restClient = RestClient.builder( + new HttpHost("localhost", 9200, "http"), + new HttpHost("localhost", 9201, "http")).build()) { + RestClientActions client = restClient.withNodeSelector(NodeSelector.NOT_MASTER); + client.performRequest("POST", "/test_index/test_type", Collections.emptyMap(), + new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON)); + } } @SuppressWarnings("unused") diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java index 34a4988358653..3070af3176c83 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java @@ -41,7 +41,7 @@ /** * Class responsible for sniffing the http hosts from elasticsearch through the nodes info api and returning them back. - * Compatible with elasticsearch 5.x and 2.x. + * Compatible with elasticsearch 2.x+. */ public final class ElasticsearchHostsSniffer implements HostsSniffer { From 753fb864264d618bafe9e3c59991812da9f31fc4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 21 Mar 2018 13:40:16 -0400 Subject: [PATCH 02/27] Further Adds selector support to tests, plumbs selectors some more. --- .../client/RestHighLevelClient.java | 2 + .../client/AbstractRestClientActions.java | 6 +- .../elasticsearch/client/HostMetadata.java | 143 ++++++++++++++++ .../{NodeSelector.java => HostSelector.java} | 18 +- .../java/org/elasticsearch/client/Node.java | 78 --------- .../org/elasticsearch/client/Response.java | 2 +- .../org/elasticsearch/client/RestClient.java | 158 ++++++++++-------- .../client/RestClientActions.java | 4 +- .../client/RestClientBuilder.java | 10 +- .../elasticsearch/client/RestClientView.java | 35 ++-- .../client/RestClientMultipleHostsTests.java | 53 +++++- .../client/RestClientSingleHostTests.java | 29 +++- .../elasticsearch/client/RestClientTests.java | 6 +- .../RestClientDocumentation.java | 6 +- .../sniff/ElasticsearchHostsSniffer.java | 57 +++++-- .../client/sniff/HostsSniffer.java | 10 +- .../elasticsearch/client/sniff/Sniffer.java | 14 +- .../sniff/ElasticsearchHostsSnifferTests.java | 61 +++---- .../client/sniff/MockHostsSniffer.java | 8 +- .../documentation/SnifferDocumentation.java | 4 +- test/framework/build.gradle | 1 + .../test/rest/ESRestTestCase.java | 35 ++++ .../rest/yaml/ClientYamlDocsTestClient.java | 14 +- .../test/rest/yaml/ClientYamlTestClient.java | 10 +- .../yaml/ClientYamlTestExecutionContext.java | 27 ++- .../rest/yaml/ESClientYamlSuiteTestCase.java | 6 +- .../test/rest/yaml/Features.java | 1 + .../test/rest/yaml/parser/package-info.java | 24 --- .../yaml/section/ClientYamlTestSection.java | 9 +- .../test/rest/yaml/section/DoSection.java | 55 +++++- .../test/rest/yaml/section/SkipSection.java | 2 +- 31 files changed, 603 insertions(+), 285 deletions(-) create mode 100644 client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java rename client/rest/src/main/java/org/elasticsearch/client/{NodeSelector.java => HostSelector.java} (69%) delete mode 100644 client/rest/src/main/java/org/elasticsearch/client/Node.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/package-info.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 85d79f2af6ccf..dbd94c8e0c41b 100755 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -196,6 +196,8 @@ public RestHighLevelClient(RestClientBuilder restClientBuilder) { this(restClientBuilder, Collections.emptyList()); } + // NOCOMMIT revert so we can use the same wrapping pattern here? + /** * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the * {@link RestClient} to be used to perform requests and parsers for custom response sections added to Elasticsearch through plugins. diff --git a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java index 52ef58f011e34..8ee37ce25199c 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java @@ -37,12 +37,12 @@ import org.apache.http.conn.ConnectTimeoutException; abstract class AbstractRestClientActions implements RestClientActions { - protected abstract void performRequestAsyncNoCatch(String method, String endpoint, Map params, + abstract SyncResponseListener syncResponseListener(); + + abstract void performRequestAsyncNoCatch(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, ResponseListener responseListener, Header[] headers); - protected abstract SyncResponseListener syncResponseListener(); - /** * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without parameters diff --git a/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java b/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java new file mode 100644 index 0000000000000..9f81c793df3f2 --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java @@ -0,0 +1,143 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.util.Objects; + +import org.apache.http.HttpHost; + +public class HostMetadata { + public static final HostMetadataResolver EMPTY_RESOLVER = new HostMetadataResolver() { + @Override + public HostMetadata resolveMetadata(HttpHost host) { + return null; + } + }; + /** + * Look up metadata about the provided host. Implementers should not make network + * calls, instead, they should look up previously fetched data. See Elasticsearch's + * Sniffer for an example implementation. + */ + public interface HostMetadataResolver { + /** + * @return {@link HostMetadat} about the provided host if we have any + * metadata, {@code null} otherwise + */ + HostMetadata resolveMetadata(HttpHost host); + } + + private final String version; + private final Roles roles; + + public HostMetadata(String version, Roles roles) { + this.version = Objects.requireNonNull(version, "version is required"); + this.roles = Objects.requireNonNull(roles, "roles is required"); + } + + /** + * Version of the node. + */ + public String version() { + return version; + } + + /** + * Roles the node is implementing. + */ + public Roles roles() { + return roles; + } + + @Override + public String toString() { + return "[version=" + version + ", roles=" + roles + "]"; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + HostMetadata other = (HostMetadata) obj; + return version.equals(other.version) + && roles.equals(other.roles); + } + + @Override + public int hashCode() { + return Objects.hash(version, roles); + } + + public static final class Roles { + private final boolean master; + private final boolean data; + private final boolean ingest; + + public Roles(boolean master, boolean data, boolean ingest) { + this.master = master; + this.data = data; + this.ingest = ingest; + } + + /** + * The node could be elected master. + */ + public boolean master() { + return master; + } + /** + * The node stores data. + */ + public boolean data() { + return data; + } + /** + * The node runs ingest pipelines. + */ + public boolean ingest() { + return ingest; + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(3); + if (master) result.append('m'); + if (data) result.append('d'); + if (ingest) result.append('i'); + return result.toString(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + Roles other = (Roles) obj; + return master == other.master + && data == other.data + && ingest == other.ingest; + } + + @Override + public int hashCode() { + return Objects.hash(master, data, ingest); + } + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java b/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java similarity index 69% rename from client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java rename to client/rest/src/main/java/org/elasticsearch/client/HostSelector.java index 04bd70e1aeca4..f39f352f3dd35 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java +++ b/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java @@ -19,13 +19,15 @@ package org.elasticsearch.client; -public interface NodeSelector { +import org.apache.http.HttpHost; + +public interface HostSelector { /** * Selector that matches any node. */ - NodeSelector ANY = new NodeSelector() { + HostSelector ANY = new HostSelector() { @Override - public boolean select(Node node) { + public boolean select(HttpHost host, HostMetadata meta) { return true; } }; @@ -34,16 +36,16 @@ public boolean select(Node node) { * Selector that matches any node that doesn't have the master roles * or doesn't have any information about roles. */ - NodeSelector NOT_MASTER = new NodeSelector() { + HostSelector NOT_MASTER = new HostSelector() { @Override - public boolean select(Node node) { - return node.roles() == null || false == node.roles().master(); + public boolean select(HttpHost host, HostMetadata meta) { + return meta != null && false == meta.roles().master(); } }; /** - * Return {@code true} if this node should be used for requests, {@code false} + * Return {@code true} if the provided host should be used for requests, {@code false} * otherwise. */ - boolean select(Node node); + boolean select(HttpHost host, HostMetadata meta); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/Node.java b/client/rest/src/main/java/org/elasticsearch/client/Node.java deleted file mode 100644 index 84e8ec1e310d4..0000000000000 --- a/client/rest/src/main/java/org/elasticsearch/client/Node.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client; - -import java.util.Objects; - -import org.apache.http.HttpHost; - -public final class Node { - interface Roles { - boolean master(); - boolean data(); - boolean ingest(); - } - - public static Node build(HttpHost host) { - return build(host, null); - } - - public static Node build(HttpHost host, Roles roles) { - return new Node(host, roles); - } - - private final HttpHost host; - private final Roles roles; - - private Node(HttpHost host, Roles roles) { - this.host = Objects.requireNonNull(host, "host cannot be null"); - this.roles = roles; - } - - public HttpHost host() { - return host; - } - - public Roles roles() { - return roles; - } - - @Override - public String toString() { - return host.toString(); - } - - @Override - public boolean equals(Object obj) { - if (obj == this) { - return true; - } - if (obj == null || obj.getClass() != getClass()) { - return false; - } - Node other = (Node) obj; - return host.equals(other.host); - } - - @Override - public int hashCode() { - return host.hashCode(); - } -} diff --git a/client/rest/src/main/java/org/elasticsearch/client/Response.java b/client/rest/src/main/java/org/elasticsearch/client/Response.java index 02aedb4765abe..39bbf769713b2 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/Response.java +++ b/client/rest/src/main/java/org/elasticsearch/client/Response.java @@ -40,7 +40,7 @@ public class Response { Response(RequestLine requestLine, HttpHost host, HttpResponse response) { Objects.requireNonNull(requestLine, "requestLine cannot be null"); - Objects.requireNonNull(host, "node cannot be null"); + Objects.requireNonNull(host, "host cannot be null"); Objects.requireNonNull(response, "response cannot be null"); this.requestLine = requestLine; this.host = host; diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index d30d200791770..be1ecf6389b2f 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -44,6 +44,7 @@ import org.apache.http.nio.client.methods.HttpAsyncMethods; import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; +import org.elasticsearch.client.HostMetadata.HostMetadataResolver; import java.io.Closeable; import java.io.IOException; @@ -96,18 +97,18 @@ public class RestClient extends AbstractRestClientActions implements Closeable { private final long maxRetryTimeoutMillis; private final String pathPrefix; private final AtomicInteger lastHostIndex = new AtomicInteger(0); - private volatile HostTuple> hostTuple; - private final ConcurrentMap blacklist = new ConcurrentHashMap<>(); + private final ConcurrentMap blacklist = new ConcurrentHashMap<>(); private final FailureListener failureListener; + private volatile HostTuple> hostTuple; RestClient(CloseableHttpAsyncClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders, - HttpHost[] hosts, String pathPrefix, FailureListener failureListener) { + HttpHost[] hosts, HostMetadataResolver metaResolver, String pathPrefix, FailureListener failureListener) { this.client = client; this.maxRetryTimeoutMillis = maxRetryTimeoutMillis; this.defaultHeaders = Collections.unmodifiableList(Arrays.asList(defaultHeaders)); this.failureListener = failureListener; this.pathPrefix = pathPrefix; - setHosts(hosts); + setHosts(Arrays.asList(hosts), metaResolver); } /** @@ -123,55 +124,66 @@ public static RestClientBuilder builder(HttpHost... hosts) { * @see HttpHost */ public void setHosts(HttpHost... hosts) { - if (hosts == null || hosts.length == 0) { - throw new IllegalArgumentException("hosts must not be null nor empty"); + if (hosts == null) { + throw new IllegalArgumentException("hosts must not be null"); } - Node[] nodes = new Node[hosts.length]; - for (int i = 0; i < hosts.length; i++) { - final HttpHost host = hosts[i]; - nodes[i] = Node.build(host); - } - setNodes(nodes); + setHosts(Arrays.asList(hosts), hostTuple.metaResolver); } - public synchronized void setNodes(Node... nodes) { - if (nodes == null || nodes.length == 0) { - throw new IllegalArgumentException("nodes must not be null nor empty"); + /** + * Replaces the hosts that the client communicates with and the + * {@link HostMetadata} used by any {@link HostSelector}s. + * @see HttpHost + */ + public void setHosts(Iterable hosts, HostMetadataResolver metaResolver) { + if (hosts == null) { + throw new IllegalArgumentException("hosts must not be null"); } - Set newNodes = new HashSet<>(); + if (metaResolver == null) { + throw new IllegalArgumentException("metaResolver must not be null"); + } + Set newHosts = new HashSet<>(); AuthCache authCache = new BasicAuthCache(); - for (Node node : nodes) { - Objects.requireNonNull(node, "node cannot be null"); - newNodes.add(node); - authCache.put(node.host(), new BasicScheme()); + + for (HttpHost host : hosts) { + Objects.requireNonNull(host, "host cannot be null"); + newHosts.add(host); + authCache.put(host, new BasicScheme()); } - this.hostTuple = new HostTuple<>(Collections.unmodifiableSet(newNodes), authCache); + if (newHosts.isEmpty()) { + throw new IllegalArgumentException("hosts must not be empty"); + } + this.hostTuple = new HostTuple<>(Collections.unmodifiableSet(newHosts), authCache, metaResolver); this.blacklist.clear(); } - @Override // NOCOMMIT this shouldn't be public - public SyncResponseListener syncResponseListener() { + public HostMetadataResolver getHostMetadataResolver() { + return hostTuple.metaResolver; + } + + @Override + final SyncResponseListener syncResponseListener() { return new SyncResponseListener(maxRetryTimeoutMillis); } @Override - protected void performRequestAsyncNoCatch(String method, String endpoint, Map params, - HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, Header[] headers) { - // Requests made directly to the client use the noop NodeSelector. - NodeSelector nodeSelector = NodeSelector.ANY; - performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, - responseListener, nodeSelector, headers); + public RestClientActions withHostSelector(HostSelector hostSelector) { + return new RestClientView(this, hostSelector); } @Override - public RestClientActions withNodeSelector(NodeSelector nodeSelector) { - return new RestClientView(this, nodeSelector); + final void performRequestAsyncNoCatch(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + ResponseListener responseListener, Header[] headers) { + // Requests made directly to the client use the noop HostSelector. + HostSelector hostSelector = HostSelector.ANY; + performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, + responseListener, hostSelector, headers); } void performRequestAsyncNoCatch(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, NodeSelector nodeSelector, Header[] headers) { + ResponseListener responseListener, HostSelector hostSelector, Header[] headers) { Objects.requireNonNull(params, "params must not be null"); Map requestParams = new HashMap<>(params); //ignore is a special parameter supported by the clients, shouldn't be sent to es @@ -204,17 +216,17 @@ void performRequestAsyncNoCatch(String method, String endpoint, Map> hostTuple, final HttpRequestBase request, + private void performRequestAsync(final long startTime, final HostTuple> hostTuple, final HttpRequestBase request, final Set ignoreErrorCodes, final HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, final FailureTrackingResponseListener listener) { - final Node node = hostTuple.hosts.next(); + final HttpHost host = hostTuple.hosts.next(); //we stream the request body if the entity allows for it - final HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(node.host(), request); + final HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request); final HttpAsyncResponseConsumer asyncResponseConsumer = httpAsyncResponseConsumerFactory.createHttpAsyncResponseConsumer(); final HttpClientContext context = HttpClientContext.create(); @@ -223,21 +235,21 @@ private void performRequestAsync(final long startTime, final HostTuple> nextHost(NodeSelector nodeSelector) { - final HostTuple> hostTuple = this.hostTuple; - Collection nextHosts = Collections.emptySet(); + private HostTuple> nextHost(HostSelector hostSelector) { + final HostTuple> hostTuple = this.hostTuple; + Collection nextHosts = Collections.emptySet(); do { - Set filteredHosts = new HashSet<>(hostTuple.hosts); - for (Iterator hostItr = filteredHosts.iterator(); hostItr.hasNext();) { - final Node node = hostItr.next(); - if (false == nodeSelector.select(node)) { + Set filteredHosts = new HashSet<>(hostTuple.hosts); + for (Iterator hostItr = filteredHosts.iterator(); hostItr.hasNext();) { + final HttpHost host = hostItr.next(); + if (false == hostSelector.select(host, hostTuple.metaResolver.resolveMetadata(host))) { hostItr.remove(); } } - for (Map.Entry entry : blacklist.entrySet()) { + for (Map.Entry entry : blacklist.entrySet()) { if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) { filteredHosts.remove(entry.getKey()); } @@ -328,42 +340,42 @@ private HostTuple> nextHost(NodeSelector nodeSelector) { * Last resort: If there are no good host to use, return a single dead one, * the one that's closest to being retried *and* matches the selector. */ - List> sortedHosts = new ArrayList<>(blacklist.entrySet()); + List> sortedHosts = new ArrayList<>(blacklist.entrySet()); if (sortedHosts.size() > 0) { - Collections.sort(sortedHosts, new Comparator>() { + Collections.sort(sortedHosts, new Comparator>() { @Override - public int compare(Map.Entry o1, Map.Entry o2) { + public int compare(Map.Entry o1, Map.Entry o2) { return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos()); } }); - Iterator> nodeItr = sortedHosts.iterator(); + Iterator> nodeItr = sortedHosts.iterator(); while (nodeItr.hasNext()) { - final Node deadNode = nodeItr.next().getKey(); - if (nodeSelector.select(deadNode)) { - logger.trace("resurrecting host [" + deadNode.host() + "]"); - nextHosts = Collections.singleton(deadNode); + final HttpHost deadHost = nodeItr.next().getKey(); + if (hostSelector.select(deadHost, hostTuple.metaResolver.resolveMetadata(deadHost))) { + logger.trace("resurrecting host [" + deadHost + "]"); + nextHosts = Collections.singleton(deadHost); break; } } } // NOCOMMIT we get here if the selector rejects all hosts } else { - List rotatedHosts = new ArrayList<>(filteredHosts); + List rotatedHosts = new ArrayList<>(filteredHosts); Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement()); nextHosts = rotatedHosts; } } while(nextHosts.isEmpty()); - return new HostTuple<>(nextHosts.iterator(), hostTuple.authCache); + return new HostTuple<>(nextHosts.iterator(), hostTuple.authCache, hostTuple.metaResolver); } /** * Called after each successful request call. * Receives as an argument the host that was used for the successful request. */ - private void onResponse(Node node) { - DeadHostState removedHost = this.blacklist.remove(node); + private void onResponse(HttpHost host) { + DeadHostState removedHost = this.blacklist.remove(host); if (logger.isDebugEnabled() && removedHost != null) { - logger.debug("removed host [" + node + "] from blacklist"); + logger.debug("removed host [" + host + "] from blacklist"); } } @@ -371,19 +383,19 @@ private void onResponse(Node node) { * Called after each failed attempt. * Receives as an argument the host that was used for the failed attempt. */ - private void onFailure(Node node) throws IOException { + private void onFailure(HttpHost host) throws IOException { while(true) { - DeadHostState previousDeadHostState = blacklist.putIfAbsent(node, DeadHostState.INITIAL_DEAD_STATE); + DeadHostState previousDeadHostState = blacklist.putIfAbsent(host, DeadHostState.INITIAL_DEAD_STATE); if (previousDeadHostState == null) { - logger.debug("added host [" + node + "] to blacklist"); + logger.debug("added host [" + host + "] to blacklist"); break; } - if (blacklist.replace(node, previousDeadHostState, new DeadHostState(previousDeadHostState))) { - logger.debug("updated host [" + node + "] already in blacklist"); + if (blacklist.replace(host, previousDeadHostState, new DeadHostState(previousDeadHostState))) { + logger.debug("updated host [" + host + "] already in blacklist"); break; } } - failureListener.onFailure(node.host()); + failureListener.onFailure(host); } @Override @@ -528,10 +540,12 @@ public void onFailure(HttpHost host) { private static class HostTuple { final T hosts; final AuthCache authCache; + final HostMetadataResolver metaResolver; - HostTuple(final T hosts, final AuthCache authCache) { + HostTuple(final T hosts, final AuthCache authCache, final HostMetadataResolver metaResolver) { this.hosts = hosts; this.authCache = authCache; + this.metaResolver = metaResolver; } } } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java index be9aaaa05eefb..1a2985f6ef998 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java @@ -173,9 +173,9 @@ void performRequestAsync(String method, String endpoint, Map par ResponseListener responseListener, Header... headers); /** - * Create a view that only runs requests on selected nodes. This object + * Create a client that only runs requests on selected hosts. This client * has no state of its own and backs everything to the {@link RestClient} * that created it. */ - RestClientActions withNodeSelector(NodeSelector nodeSelector); + RestClientActions withHostSelector(HostSelector hostSelector); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java index 286ed7dd53910..5fd5afd6f89ea 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java @@ -27,6 +27,7 @@ import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.nio.conn.SchemeIOSessionStrategy; +import org.elasticsearch.client.HostMetadata.HostMetadataResolver; import javax.net.ssl.SSLContext; import java.security.AccessController; @@ -50,6 +51,7 @@ public final class RestClientBuilder { private static final Header[] EMPTY_HEADERS = new Header[0]; private final HttpHost[] hosts; + private HostMetadataResolver metaResolver = HostMetadata.EMPTY_RESOLVER; private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT_MILLIS; private Header[] defaultHeaders = EMPTY_HEADERS; private RestClient.FailureListener failureListener; @@ -74,6 +76,11 @@ public final class RestClientBuilder { this.hosts = hosts; } + public RestClientBuilder setHostMetadata(HostMetadataResolver metaResolver) { + this.metaResolver = metaResolver; + return this; + } + /** * Sets the default request headers, which will be sent along with each request. *

@@ -187,7 +194,8 @@ public CloseableHttpAsyncClient run() { return createHttpClient(); } }); - RestClient restClient = new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts, pathPrefix, failureListener); + RestClient restClient = new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts, + metaResolver, pathPrefix, failureListener); httpClient.start(); return restClient; } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java index be246d8f31f3a..9838a14a46feb 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java @@ -23,41 +23,42 @@ import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; /** * Light weight view into a {@link RestClient} that doesn't have any state of its own. */ class RestClientView extends AbstractRestClientActions { private final RestClient delegate; - private final NodeSelector nodeSelector; + private final HostSelector hostSelector; - protected RestClientView(RestClient delegate, NodeSelector nodeSelector) { + protected RestClientView(RestClient delegate, HostSelector hostSelector) { this.delegate = delegate; - this.nodeSelector = nodeSelector; + this.hostSelector = hostSelector; } @Override - protected final void performRequestAsyncNoCatch(String method, String endpoint, Map params, - HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, Header[] headers) { - delegate.performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, - responseListener, nodeSelector, headers); - } - - @Override - protected final SyncResponseListener syncResponseListener() { + final SyncResponseListener syncResponseListener() { return delegate.syncResponseListener(); } @Override - public final RestClientView withNodeSelector(final NodeSelector nodeSelector) { - final NodeSelector inner = this.nodeSelector; - NodeSelector combo = new NodeSelector() { + public final RestClientView withHostSelector(final HostSelector hostSelector) { + final HostSelector inner = this.hostSelector; + HostSelector combo = new HostSelector() { @Override - public boolean select(Node node) { - return inner.select(node) && nodeSelector.select(node); + public boolean select(HttpHost host, HostMetadata meta) { + return inner.select(host, meta) && hostSelector.select(host, meta); } }; return new RestClientView(delegate, combo); } + + @Override + final void performRequestAsyncNoCatch(String method, String endpoint, Map params, + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + ResponseListener responseListener, Header[] headers) { + delegate.performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, + responseListener, hostSelector, headers); + } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java index a3a834ff3204b..7a39ccc94ae10 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java @@ -35,6 +35,7 @@ import org.apache.http.message.BasicStatusLine; import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; +import org.elasticsearch.client.HostMetadata.HostMetadataResolver; import org.junit.After; import org.junit.Before; import org.mockito.invocation.InvocationOnMock; @@ -42,8 +43,10 @@ import java.io.IOException; import java.net.SocketTimeoutException; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -56,6 +59,7 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -63,6 +67,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import static java.util.Collections.singletonMap; + /** * Tests for {@link RestClient} behaviour against multiple hosts: fail-over, blacklisting etc. * Relies on a mock http client to intercept requests and return desired responses based on request path. @@ -70,6 +76,7 @@ public class RestClientMultipleHostsTests extends RestClientTestCase { private ExecutorService exec = Executors.newFixedThreadPool(1); + private volatile Map hostMetadata = Collections.emptyMap(); private RestClient restClient; private HttpHost[] httpHosts; private HostsTrackingFailureListener failureListener; @@ -113,8 +120,17 @@ public void run() { for (int i = 0; i < numHosts; i++) { httpHosts[i] = new HttpHost("localhost", 9200 + i); } + /* + * Back the metadata to a map that we can manipulate during testing. + */ + HostMetadataResolver metaResolver = new HostMetadataResolver() { + @Override + public HostMetadata resolveMetadata(HttpHost host) { + return hostMetadata.get(host); + } + }; failureListener = new HostsTrackingFailureListener(); - restClient = new RestClient(httpClient, 10000, new Header[0], httpHosts, null, failureListener); + restClient = new RestClient(httpClient, 10000, new Header[0], httpHosts, metaResolver, null, failureListener); } /** @@ -308,6 +324,41 @@ public void testRoundRobinRetryErrors() throws IOException { } } + /** + * Test that calling {@link RestClient#setHosts(Iterable, HostMetadataResolver)} + * sets the {@link HostMetadataResolver}. + */ + public void testSetHostWithMetadataResolver() throws IOException { + HostMetadataResolver firstPositionIsClient = new HostMetadataResolver() { + @Override + public HostMetadata resolveMetadata(HttpHost host) { + HostMetadata.Roles roles; + if (host == httpHosts[0]) { + roles = new HostMetadata.Roles(false, false, false); + } else { + roles = new HostMetadata.Roles(true, true, true); + } + return new HostMetadata("dummy", roles); + } + }; + restClient.setHosts(Arrays.asList(httpHosts), firstPositionIsClient); + assertSame(firstPositionIsClient, restClient.getHostMetadataResolver()); + Response response = restClient.withHostSelector(HostSelector.NOT_MASTER).performRequest("GET", "/200"); + assertEquals(httpHosts[0], response.getHost()); + } + + /** + * Test that calling {@link RestClient#setHosts(Iterable)} preserves the + * {@link HostMetadataResolver}. + */ + public void testSetHostWithoutMetadataResolver() throws IOException { + HttpHost expected = randomFrom(httpHosts); + hostMetadata = singletonMap(expected, new HostMetadata("dummy", new HostMetadata.Roles(false, false, false))); + restClient.setHosts(httpHosts); + Response response = restClient.withHostSelector(HostSelector.NOT_MASTER).performRequest("GET", "/200"); + assertEquals(expected, response.getHost()); + } + private static String randomErrorRetryEndpoint() { switch(RandomNumbers.randomIntBetween(getRandom(), 0, 3)) { case 0: diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java index caf9ce6be2e07..50a701dc3f56a 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java @@ -47,6 +47,7 @@ import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.HostMetadata.HostMetadataResolver; import org.junit.After; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -96,6 +97,7 @@ public class RestClientSingleHostTests extends RestClientTestCase { private RestClient restClient; private Header[] defaultHeaders; private HttpHost httpHost; + private volatile HostMetadata hostMetadata; private CloseableHttpAsyncClient httpClient; private HostsTrackingFailureListener failureListener; @@ -149,7 +151,14 @@ public void run() { defaultHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header-default"); httpHost = new HttpHost("localhost", 9200); failureListener = new HostsTrackingFailureListener(); - restClient = new RestClient(httpClient, 10000, defaultHeaders, new HttpHost[]{httpHost}, null, failureListener); + HostMetadataResolver metaResolver = new HostMetadataResolver(){ + @Override + public HostMetadata resolveMetadata(HttpHost host) { + return hostMetadata; + } + }; + restClient = new RestClient(httpClient, 10000, defaultHeaders, new HttpHost[]{httpHost}, + metaResolver, null, failureListener); } /** @@ -196,18 +205,18 @@ public void testInternalHttpRequest() throws Exception { } } - public void testSetHosts() throws IOException { + public void testSetHostsFailures() throws IOException { try { restClient.setHosts((HttpHost[]) null); fail("setHosts should have failed"); } catch (IllegalArgumentException e) { - assertEquals("hosts must not be null nor empty", e.getMessage()); + assertEquals("hosts must not be null", e.getMessage()); } try { restClient.setHosts(); fail("setHosts should have failed"); } catch (IllegalArgumentException e) { - assertEquals("hosts must not be null nor empty", e.getMessage()); + assertEquals("hosts must not be empty", e.getMessage()); } try { restClient.setHosts((HttpHost) null); @@ -221,6 +230,18 @@ public void testSetHosts() throws IOException { } catch (NullPointerException e) { assertEquals("host cannot be null", e.getMessage()); } + try { + restClient.setHosts(null, HostMetadata.EMPTY_RESOLVER); + fail("setHosts should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("hosts must not be null", e.getMessage()); + } + try { + restClient.setHosts(Arrays.asList(new HttpHost("localhost", 9200)), null); + fail("setHosts should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("metaResolver must not be null", e.getMessage()); + } } /** diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java index 33323d39663e2..142a43bde140d 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java @@ -41,7 +41,8 @@ public class RestClientTests extends RestClientTestCase { public void testCloseIsIdempotent() throws IOException { HttpHost[] hosts = new HttpHost[]{new HttpHost("localhost", 9200)}; CloseableHttpAsyncClient closeableHttpAsyncClient = mock(CloseableHttpAsyncClient.class); - RestClient restClient = new RestClient(closeableHttpAsyncClient, 1_000, new Header[0], hosts, null, null); + RestClient restClient = new RestClient(closeableHttpAsyncClient, 1_000, new Header[0], + hosts, HostMetadata.EMPTY_RESOLVER, null, null); restClient.close(); verify(closeableHttpAsyncClient, times(1)).close(); restClient.close(); @@ -149,6 +150,7 @@ public void testBuildUriLeavesPathUntouched() { private static RestClient createRestClient() { HttpHost[] hosts = new HttpHost[]{new HttpHost("localhost", 9200)}; - return new RestClient(mock(CloseableHttpAsyncClient.class), randomLongBetween(1_000, 30_000), new Header[]{}, hosts, null, null); + return new RestClient(mock(CloseableHttpAsyncClient.class), randomLongBetween(1_000, 30_000), + new Header[] {}, hosts, HostMetadata.EMPTY_RESOLVER, null, null); } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java index 085ed06e43935..dd4939b4e35cf 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java +++ b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java @@ -37,8 +37,8 @@ import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.HostSelector; import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; -import org.elasticsearch.client.NodeSelector; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseListener; import org.elasticsearch.client.RestClient; @@ -258,11 +258,11 @@ public void onFailure(Exception exception) { } } - public void testNodeSelector() throws IOException { + public void testHostSelector() throws IOException { try (RestClient restClient = RestClient.builder( new HttpHost("localhost", 9200, "http"), new HttpHost("localhost", 9201, "http")).build()) { - RestClientActions client = restClient.withNodeSelector(NodeSelector.NOT_MASTER); + RestClientActions client = restClient.withHostSelector(HostSelector.NOT_MASTER); client.performRequest("POST", "/test_index/test_type", Collections.emptyMap(), new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON)); } diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java index 3070af3176c83..e557b738c2c7b 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java @@ -26,15 +26,16 @@ import org.apache.commons.logging.LogFactory; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; +import org.elasticsearch.client.HostMetadata; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.HostMetadata.Roles; import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; +import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -89,18 +90,19 @@ public ElasticsearchHostsSniffer(RestClient restClient, long sniffRequestTimeout /** * Calls the elasticsearch nodes info api, parses the response and returns all the found http hosts */ - public List sniffHosts() throws IOException { + @Override + public Map sniffHosts() throws IOException { Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams); return readHosts(response.getEntity()); } - private List readHosts(HttpEntity entity) throws IOException { + private Map readHosts(HttpEntity entity) throws IOException { try (InputStream inputStream = entity.getContent()) { JsonParser parser = jsonFactory.createParser(inputStream); if (parser.nextToken() != JsonToken.START_OBJECT) { throw new IOException("expected data to start with an object"); } - List hosts = new ArrayList<>(); + Map hosts = new HashMap<>(); while (parser.nextToken() != JsonToken.END_OBJECT) { if (parser.getCurrentToken() == JsonToken.START_OBJECT) { if ("nodes".equals(parser.getCurrentName())) { @@ -108,11 +110,7 @@ private List readHosts(HttpEntity entity) throws IOException { JsonToken token = parser.nextToken(); assert token == JsonToken.START_OBJECT; String nodeId = parser.getCurrentName(); - HttpHost sniffedHost = readHost(nodeId, parser, this.scheme); - if (sniffedHost != null) { - logger.trace("adding node [" + nodeId + "]"); - hosts.add(sniffedHost); - } + readHost(nodeId, parser, scheme, hosts); } } else { parser.skipChildren(); @@ -123,9 +121,15 @@ private List readHosts(HttpEntity entity) throws IOException { } } - private static HttpHost readHost(String nodeId, JsonParser parser, Scheme scheme) throws IOException { + private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Map hosts) throws IOException { + // NOCOMMIT test me against 2.x and 5.x HttpHost httpHost = null; String fieldName = null; + String version = null; + boolean sawRoles = false; + boolean master = false; + boolean data = false; + boolean ingest = false; while (parser.nextToken() != JsonToken.END_OBJECT) { if (parser.getCurrentToken() == JsonToken.FIELD_NAME) { fieldName = parser.getCurrentName(); @@ -143,14 +147,41 @@ private static HttpHost readHost(String nodeId, JsonParser parser, Scheme scheme } else { parser.skipChildren(); } + } else if (parser.currentToken() == JsonToken.START_ARRAY) { + if ("roles".equals(fieldName)) { + sawRoles = true; + while (parser.nextToken() != JsonToken.END_ARRAY) { + switch (parser.getText()) { + case "master": + master = true; + break; + case "data": + data = true; + break; + case "ingest": + ingest = true; + break; + default: + logger.warn("unknown role [" + parser.getText() + "] on node [" + nodeId + "]"); + } + } + } else { + parser.skipChildren(); + } + } else if (parser.currentToken().isScalarValue()) { + if ("version".equals(fieldName)) { + version = parser.getText(); + } } } //http section is not present if http is not enabled on the node, ignore such nodes if (httpHost == null) { logger.debug("skipping node [" + nodeId + "] with http disabled"); - return null; + } else { + logger.trace("adding node [" + nodeId + "]"); + assert sawRoles : "didn't see roles for [" + nodeId + "]"; + hosts.put(httpHost, new HostMetadata(version, new Roles(master, data, ingest))); } - return httpHost; } public enum Scheme { diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java index 9eb7b34425944..95902169ca970 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java @@ -20,16 +20,18 @@ package org.elasticsearch.client.sniff; import org.apache.http.HttpHost; +import org.elasticsearch.client.HostMetadata; import java.io.IOException; -import java.util.List; +import java.util.Map; /** - * Responsible for sniffing the http hosts + * Responsible for sniffing the http hosts. */ public interface HostsSniffer { /** - * Returns the sniffed http hosts + * Returns a {@link Map} from sniffed {@link HttpHost} to metadata + * sniffed about the host. */ - List sniffHosts() throws IOException; + Map sniffHosts() throws IOException; } diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java index c655babd9ed3d..9e68071436d4b 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java @@ -22,14 +22,18 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpHost; +import org.elasticsearch.client.HostMetadata; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.HostMetadata.HostMetadataResolver; import java.io.Closeable; import java.io.IOException; import java.security.AccessController; import java.security.PrivilegedAction; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -116,7 +120,7 @@ void sniffOnFailure(HttpHost failedHost) { void sniff(HttpHost excludeHost, long nextSniffDelayMillis) { if (running.compareAndSet(false, true)) { try { - List sniffedHosts = hostsSniffer.sniffHosts(); + final Map sniffedHosts = hostsSniffer.sniffHosts(); logger.debug("sniffed hosts: " + sniffedHosts); if (excludeHost != null) { sniffedHosts.remove(excludeHost); @@ -124,7 +128,13 @@ void sniff(HttpHost excludeHost, long nextSniffDelayMillis) { if (sniffedHosts.isEmpty()) { logger.warn("no hosts to set, hosts will be updated at the next sniffing round"); } else { - this.restClient.setHosts(sniffedHosts.toArray(new HttpHost[sniffedHosts.size()])); + HostMetadataResolver resolver = new HostMetadataResolver() { + @Override + public HostMetadata resolveMetadata(HttpHost host) { + return sniffedHosts.get(host); + } + }; + this.restClient.setHosts(sniffedHosts.keySet(), resolver); } } catch (Exception e) { logger.error("error while sniffing nodes", e); diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java index 483b7df62f95a..3f3237c2eb91e 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java @@ -31,6 +31,7 @@ import org.apache.http.HttpHost; import org.apache.http.client.methods.HttpGet; import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement; +import org.elasticsearch.client.HostMetadata; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; @@ -45,6 +46,7 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -117,15 +119,11 @@ public void testSniffNodes() throws IOException { try (RestClient restClient = RestClient.builder(httpHost).build()) { ElasticsearchHostsSniffer sniffer = new ElasticsearchHostsSniffer(restClient, sniffRequestTimeout, scheme); try { - List sniffedHosts = sniffer.sniffHosts(); + Map sniffedHosts = sniffer.sniffHosts(); if (sniffResponse.isFailure) { fail("sniffNodes should have failed"); } - assertThat(sniffedHosts.size(), equalTo(sniffResponse.hosts.size())); - Iterator responseHostsIterator = sniffResponse.hosts.iterator(); - for (HttpHost sniffedHost : sniffedHosts) { - assertEquals(sniffedHost, responseHostsIterator.next()); - } + assertEquals(sniffResponse.hosts, sniffedHosts); } catch(ResponseException e) { Response response = e.getResponse(); if (sniffResponse.isFailure) { @@ -180,7 +178,7 @@ public void handle(HttpExchange httpExchange) throws IOException { private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme scheme) throws IOException { int numNodes = RandomNumbers.randomIntBetween(getRandom(), 1, 5); - List hosts = new ArrayList<>(numNodes); + Map hosts = new HashMap<>(numNodes); JsonFactory jsonFactory = new JsonFactory(); StringWriter writer = new StringWriter(); JsonGenerator generator = jsonFactory.createGenerator(writer); @@ -195,6 +193,12 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme generator.writeObjectFieldStart("nodes"); for (int i = 0; i < numNodes; i++) { String nodeId = RandomStrings.randomAsciiOfLengthBetween(getRandom(), 5, 10); + String host = "host" + i; + int port = RandomNumbers.randomIntBetween(getRandom(), 9200, 9299); + HttpHost httpHost = new HttpHost(host, port, scheme.toString()); + HostMetadata metadata = new HostMetadata(randomAsciiAlphanumOfLength(5), + new HostMetadata.Roles(randomBoolean(), randomBoolean(), randomBoolean())); + generator.writeObjectFieldStart(nodeId); if (getRandom().nextBoolean()) { generator.writeObjectFieldStart("bogus_object"); @@ -208,10 +212,7 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme } boolean isHttpEnabled = rarely() == false; if (isHttpEnabled) { - String host = "host" + i; - int port = RandomNumbers.randomIntBetween(getRandom(), 9200, 9299); - HttpHost httpHost = new HttpHost(host, port, scheme.toString()); - hosts.add(httpHost); + hosts.put(httpHost, metadata); generator.writeObjectFieldStart("http"); if (getRandom().nextBoolean()) { generator.writeArrayFieldStart("bound_address"); @@ -230,22 +231,26 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme } generator.writeEndObject(); } - if (getRandom().nextBoolean()) { - String[] roles = {"master", "data", "ingest"}; - int numRoles = RandomNumbers.randomIntBetween(getRandom(), 0, 3); - Set nodeRoles = new HashSet<>(numRoles); - for (int j = 0; j < numRoles; j++) { - String role; - do { - role = RandomPicks.randomFrom(getRandom(), roles); - } while(nodeRoles.add(role) == false); + + List roles = Arrays.asList(new String[] {"master", "data", "ingest"}); + Collections.shuffle(roles, getRandom()); + generator.writeArrayFieldStart("roles"); + for (String role : roles) { + if ("master".equals(role) && metadata.roles().master()) { + generator.writeString("master"); } - generator.writeArrayFieldStart("roles"); - for (String nodeRole : nodeRoles) { - generator.writeString(nodeRole); + if ("data".equals(role) && metadata.roles().data()) { + generator.writeString("data"); + } + if ("ingest".equals(role) && metadata.roles().ingest()) { + generator.writeString("ingest"); } - generator.writeEndArray(); } + generator.writeEndArray(); + + generator.writeFieldName("version"); + generator.writeString(metadata.version()); + int numAttributes = RandomNumbers.randomIntBetween(getRandom(), 0, 3); Map attributes = new HashMap<>(numAttributes); for (int j = 0; j < numAttributes; j++) { @@ -271,10 +276,10 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme private static class SniffResponse { private final String nodesInfoBody; private final int nodesInfoResponseCode; - private final List hosts; + private final Map hosts; private final boolean isFailure; - SniffResponse(String nodesInfoBody, List hosts, boolean isFailure) { + SniffResponse(String nodesInfoBody, Map hosts, boolean isFailure) { this.nodesInfoBody = nodesInfoBody; this.hosts = hosts; this.isFailure = isFailure; @@ -286,10 +291,10 @@ private static class SniffResponse { } static SniffResponse buildFailure() { - return new SniffResponse("", Collections.emptyList(), true); + return new SniffResponse("", Collections.emptyMap(), true); } - static SniffResponse buildResponse(String nodesInfoBody, List hosts) { + static SniffResponse buildResponse(String nodesInfoBody, Map hosts) { return new SniffResponse(nodesInfoBody, hosts, false); } } diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java index 5a52151d76e01..a14ddcb683b79 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java @@ -20,17 +20,19 @@ package org.elasticsearch.client.sniff; import org.apache.http.HttpHost; +import org.elasticsearch.client.HostMetadata; import java.io.IOException; import java.util.Collections; -import java.util.List; +import java.util.Map; /** * Mock implementation of {@link HostsSniffer}. Useful to prevent any connection attempt while testing builders etc. */ class MockHostsSniffer implements HostsSniffer { @Override - public List sniffHosts() throws IOException { - return Collections.singletonList(new HttpHost("localhost", 9200)); + public Map sniffHosts() throws IOException { + return Collections.singletonMap(new HttpHost("localhost", 9200), + new HostMetadata("mock version", new HostMetadata.Roles(false, false, false))); } } diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java index 199632d478f81..0b541de7b8ebb 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java @@ -20,6 +20,7 @@ package org.elasticsearch.client.sniff.documentation; import org.apache.http.HttpHost; +import org.elasticsearch.client.HostMetadata; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer; import org.elasticsearch.client.sniff.HostsSniffer; @@ -28,6 +29,7 @@ import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; /** @@ -119,7 +121,7 @@ public void testUsage() throws IOException { .build(); HostsSniffer hostsSniffer = new HostsSniffer() { @Override - public List sniffHosts() throws IOException { + public Map sniffHosts() throws IOException { return null; // <1> } }; diff --git a/test/framework/build.gradle b/test/framework/build.gradle index 193fcb30988c6..5f1bc524da599 100644 --- a/test/framework/build.gradle +++ b/test/framework/build.gradle @@ -21,6 +21,7 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks; dependencies { compile "org.elasticsearch.client:elasticsearch-rest-client:${version}" + compile "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}" compile "org.elasticsearch:elasticsearch-nio:${version}" compile "org.elasticsearch:elasticsearch:${version}" compile "org.elasticsearch:elasticsearch-cli:${version}" diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index befc21eb1f697..27c4fe806fc1b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -32,10 +32,13 @@ import org.apache.http.ssl.SSLContexts; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.HostMetadata; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; +import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; @@ -423,6 +426,7 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE final TimeValue socketTimeout = TimeValue.parseTimeValue(socketTimeoutString, CLIENT_SOCKET_TIMEOUT); builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); } + return builder.build(); } @@ -532,4 +536,35 @@ protected static Map getAsMap(final String endpoint) throws IOEx assertNotNull(responseEntity); return responseEntity; } + + /** + * Sniff the cluster for host metadata if it hasn't already been sniffed. This isn't the + * same thing as using the {@link Sniffer} because: + *

    + *
  • It doesn't replace the hosts that that {@link #client} communicates with + *
  • It only runs once + *
+ */ + protected void sniffHostMetadata(RestClient client) throws IOException { + if (HostMetadata.EMPTY_RESOLVER != client.getHostMetadataResolver()) { + // Already added a resolver + return; + } + // No resolver, sniff one time and resolve metadata against the results + ElasticsearchHostsSniffer.Scheme scheme; + switch (getProtocol()) { + case "http": + scheme = ElasticsearchHostsSniffer.Scheme.HTTP; + break; + case "https": + scheme = ElasticsearchHostsSniffer.Scheme.HTTPS; + break; + default: + throw new UnsupportedOperationException("unknown protocol [" + getProtocol() + "]"); + } + ElasticsearchHostsSniffer sniffer = new ElasticsearchHostsSniffer( + adminClient, ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT, scheme); + Map meta = sniffer.sniffHosts(); + client.setHosts(clusterHosts, meta::get); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java index dacd67ccadc32..c69ad4057c6d3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java @@ -22,9 +22,11 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.elasticsearch.Version; +import org.elasticsearch.client.HostSelector; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestSpec; import java.io.IOException; @@ -40,14 +42,14 @@ */ public final class ClientYamlDocsTestClient extends ClientYamlTestClient { - public ClientYamlDocsTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List hosts, Version esVersion) - throws IOException { + public ClientYamlDocsTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List hosts, + Version esVersion) throws IOException { super(restSpec, restClient, hosts, esVersion); } - public ClientYamlTestResponse callApi(String apiName, Map params, HttpEntity entity, Map headers) - throws IOException { - + @Override + public ClientYamlTestResponse callApi(String apiName, Map params, HttpEntity entity, + Map headers, HostSelector hostSelector) throws IOException { if ("raw".equals(apiName)) { // Raw requests are bit simpler.... Map queryStringParams = new HashMap<>(params); @@ -61,6 +63,6 @@ public ClientYamlTestResponse callApi(String apiName, Map params throw new ClientYamlTestResponseException(e); } } - return super.callApi(apiName, params, entity, headers); + return super.callApi(apiName, params, entity, headers, hostSelector); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java index f5e834aa90c69..36e09364badd1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java @@ -28,9 +28,11 @@ import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; +import org.elasticsearch.client.HostSelector; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestApi; import org.elasticsearch.test.rest.yaml.restspec.ClientYamlSuiteRestPath; @@ -75,8 +77,8 @@ public Version getEsVersion() { /** * Calls an api with the provided parameters and body */ - public ClientYamlTestResponse callApi(String apiName, Map params, HttpEntity entity, Map headers) - throws IOException { + public ClientYamlTestResponse callApi(String apiName, Map params, HttpEntity entity, + Map headers, HostSelector hostSelector) throws IOException { ClientYamlSuiteRestApi restApi = restApi(apiName); @@ -170,7 +172,9 @@ public ClientYamlTestResponse callApi(String apiName, Map params logger.debug("calling api [{}]", apiName); try { - Response response = restClient.performRequest(requestMethod, requestPath, queryStringParams, entity, requestHeaders); + Response response = restClient + .withHostSelector(hostSelector) + .performRequest(requestMethod, requestPath, queryStringParams, entity, requestHeaders); return new ClientYamlTestResponse(response); } catch(ResponseException e) { throw new ClientYamlTestResponseException(e); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index ca04c0c53d12a..909e9b4c5dd50 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -25,6 +25,8 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; +import org.elasticsearch.client.HostSelector; +import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -57,9 +59,13 @@ public class ClientYamlTestExecutionContext { private final boolean randomizeContentType; - ClientYamlTestExecutionContext(ClientYamlTestClient clientYamlTestClient, boolean randomizeContentType) { + private final CheckedRunnable setHostMetadata; + + ClientYamlTestExecutionContext(ClientYamlTestClient clientYamlTestClient, CheckedRunnable setHostMetadata, + boolean randomizeContentType) { this.clientYamlTestClient = clientYamlTestClient; this.randomizeContentType = randomizeContentType; + this.setHostMetadata = setHostMetadata; } /** @@ -68,6 +74,15 @@ public class ClientYamlTestExecutionContext { */ public ClientYamlTestResponse callApi(String apiName, Map params, List> bodies, Map headers) throws IOException { + return callApi(apiName, params, bodies, headers, HostSelector.ANY); + } + + /** + * Calls an elasticsearch api with the parameters and request body provided as arguments. + * Saves the obtained response in the execution context. + */ + public ClientYamlTestResponse callApi(String apiName, Map params, List> bodies, + Map headers, HostSelector hostSelector) throws IOException { //makes a copy of the parameters before modifying them for this specific request Map requestParams = new HashMap<>(params); requestParams.putIfAbsent("error_trace", "true"); // By default ask for error traces, this my be overridden by params @@ -85,9 +100,13 @@ public ClientYamlTestResponse callApi(String apiName, Map params } } + if (hostSelector != HostSelector.ANY) { + setHostMetadata.run(); + } + HttpEntity entity = createEntity(bodies, requestHeaders); try { - response = callApiInternal(apiName, requestParams, entity, requestHeaders); + response = callApiInternal(apiName, requestParams, entity, requestHeaders, hostSelector); return response; } catch(ClientYamlTestResponseException e) { response = e.getRestTestResponse(); @@ -154,8 +173,8 @@ private BytesRef bodyAsBytesRef(Map bodyAsMap, XContentType xCon // pkg-private for testing ClientYamlTestResponse callApiInternal(String apiName, Map params, - HttpEntity entity, Map headers) throws IOException { - return clientYamlTestClient.callApi(apiName, params, entity, headers); + HttpEntity entity, Map headers, HostSelector hostSelector) throws IOException { + return clientYamlTestClient.callApi(apiName, params, entity, headers, hostSelector); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 927f9b46c966a..dfc37d69466e7 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -120,8 +120,10 @@ public void initAndResetContext() throws Exception { } } ClientYamlTestClient clientYamlTestClient = initClientYamlTestClient(restSpec, restClient, hosts, esVersion); - restTestExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient, randomizeContentType()); - adminExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient, false); + restTestExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient, + () -> sniffHostMetadata(adminClient()), randomizeContentType()); + adminExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient, + () -> sniffHostMetadata(client()), false); String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null); blacklistPathMatchers = new ArrayList<>(); for (String entry : blacklist) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java index ab9be65514a96..d2ca61f4d03c1 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java @@ -39,6 +39,7 @@ public final class Features { "catch_unauthorized", "embedded_stash_key", "headers", + "host_selector", "stash_in_key", "stash_in_path", "stash_path_replace", diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/package-info.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/package-info.java deleted file mode 100644 index de63b46eff313..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/parser/package-info.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * Parses YAML test {@link org.elasticsearch.test.rest.yaml.section.ClientYamlTestSuite}s containing - * {@link org.elasticsearch.test.rest.yaml.section.ClientYamlTestSection}s. - */ -package org.elasticsearch.test.rest.yaml.parser; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index 321d22ed70aa7..0bcda1fc3f4bc 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.test.rest.yaml.section; +import org.elasticsearch.client.HostSelector; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; @@ -91,7 +92,13 @@ public void addExecutableSection(ExecutableSection executableSection) { + "runners that do not support the [warnings] section can skip the test at line [" + doSection.getLocation().lineNumber + "]"); } - } + if (HostSelector.ANY != doSection.getHostSelector() + && false == skipSection.getFeatures().contains("host_selector")) { + throw new IllegalArgumentException("Attempted to add a [do] with a [host] section without a corresponding [skip] so " + + "runners that do not support the [host_selector] section can skip the test at line [" + + doSection.getLocation().lineNumber + "]"); + } + } this.executableSections.add(executableSection); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 7c6647d65f044..6285d3bcfb690 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -19,7 +19,11 @@ package org.elasticsearch.test.rest.yaml.section; +import org.apache.http.HttpHost; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; +import org.elasticsearch.client.HostMetadata; +import org.elasticsearch.client.HostSelector; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -120,6 +124,23 @@ public static DoSection parse(XContentParser parser) throws IOException { headers.put(headerName, parser.text()); } } + } else if ("host_selector".equals(currentFieldName)) { + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + String selectorName = null; + if (token == XContentParser.Token.FIELD_NAME) { + selectorName = parser.currentName(); + } else if (token.isValue()) { + HostSelector original = doSection.getHostSelector(); + HostSelector newSelector = buildHostSelector( + parser.getTokenLocation(), selectorName, parser.text()); + doSection.setHostSelector(new HostSelector() { + @Override + public boolean select(HttpHost host, HostMetadata meta) { + return original.select(host, meta) && newSelector.select(host, meta); + } + }); + } + } } else if (currentFieldName != null) { // must be part of API call then apiCallSection = new ApiCallSection(currentFieldName); String paramName = null; @@ -160,16 +181,17 @@ public static DoSection parse(XContentParser parser) throws IOException { return doSection; } - private static final Logger logger = Loggers.getLogger(DoSection.class); private final XContentLocation location; private String catchParam; private ApiCallSection apiCallSection; private List expectedWarningHeaders = emptyList(); + private HostSelector hostSelector = HostSelector.ANY; public DoSection(XContentLocation location) { this.location = location; + } public String getCatch() { @@ -204,6 +226,20 @@ public void setExpectedWarningHeaders(List expectedWarningHeaders) { this.expectedWarningHeaders = expectedWarningHeaders; } + /** + * Selects the node on which to run this request. + */ + public HostSelector getHostSelector() { + return hostSelector; + } + + /** + * Set the selector that decides which node can run this request. + */ + public void setHostSelector(HostSelector hostSelector) { + this.hostSelector = hostSelector; + } + @Override public XContentLocation getLocation() { return location; @@ -337,4 +373,21 @@ private String formatStatusCodeMessage(ClientYamlTestResponse restTestResponse, not(equalTo(408)), not(equalTo(409))))); } + + private static HostSelector buildHostSelector(XContentLocation location, String name, String value) { + switch (name) { + case "version": + Version[] range = SkipSection.parseVersionRange(value); + return new HostSelector() { + @Override + public boolean select(HttpHost host, HostMetadata meta) { + // NOCOMMIT actually compare the version + Version current = Version.CURRENT; + return current.onOrAfter(range[0]) && current.onOrBefore(range[1]); + } + }; + default: + throw new IllegalArgumentException("unknown host_selector [" + name + "]"); + } + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java index eb1fea4b79aed..e487f8e74da3b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/SkipSection.java @@ -153,7 +153,7 @@ public boolean isEmpty() { return EMPTY.equals(this); } - private Version[] parseVersionRange(String versionRange) { + static Version[] parseVersionRange(String versionRange) { if (versionRange == null) { return new Version[] { null, null }; } From 0d9fe423a618bf68e0bfca2170e6b4f5e17d26f9 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 21 Mar 2018 17:17:24 -0400 Subject: [PATCH 03/27] host_selector Adds `host_selector` support to `do` statements in the yaml tests. Using it looks a little like: ``` --- "some example": - skip: features: host_selector - do: host_selector: version: " - 7.0.0" # same syntax as skip apiname: something: true ``` This is implemented by using `ElasticsearchHostsSniffer` in the test framework to load sniff metadata about all hosts and stick it on a `RestClient`. The `do` section contains a `HostSelector` that it hands off to the `RestClient`. The idea is to use this in mixed version tests to target a specific version of Elasticsearch. --- .../client/RestClientMultipleHostsTests.java | 4 +-- .../yaml/ClientYamlTestExecutionContext.java | 2 +- .../rest/yaml/section/ApiCallSection.java | 17 +++++++++ .../yaml/section/ClientYamlTestSection.java | 6 ++-- .../test/rest/yaml/section/DoSection.java | 34 ++++++------------ .../ClientYamlTestExecutionContextTests.java | 26 ++++++++++++-- .../section/ClientYamlTestSectionTests.java | 31 +++++++++++++++- .../rest/yaml/section/DoSectionTests.java | 36 +++++++++++++++++++ 8 files changed, 122 insertions(+), 34 deletions(-) diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java index 7a39ccc94ae10..03561b0d5524c 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java @@ -328,7 +328,7 @@ public void testRoundRobinRetryErrors() throws IOException { * Test that calling {@link RestClient#setHosts(Iterable, HostMetadataResolver)} * sets the {@link HostMetadataResolver}. */ - public void testSetHostWithMetadataResolver() throws IOException { + public void testSetHostsWithMetadataResolver() throws IOException { HostMetadataResolver firstPositionIsClient = new HostMetadataResolver() { @Override public HostMetadata resolveMetadata(HttpHost host) { @@ -351,7 +351,7 @@ public HostMetadata resolveMetadata(HttpHost host) { * Test that calling {@link RestClient#setHosts(Iterable)} preserves the * {@link HostMetadataResolver}. */ - public void testSetHostWithoutMetadataResolver() throws IOException { + public void testSetHostsWithoutMetadataResolver() throws IOException { HttpHost expected = randomFrom(httpHosts); hostMetadata = singletonMap(expected, new HostMetadata("dummy", new HostMetadata.Roles(false, false, false))); restClient.setHosts(httpHosts); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index 909e9b4c5dd50..a7b00bea86289 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -77,7 +77,7 @@ public ClientYamlTestResponse callApi(String apiName, Map params return callApi(apiName, params, bodies, headers, HostSelector.ANY); } - /** + /** * Calls an elasticsearch api with the parameters and request body provided as arguments. * Saves the obtained response in the execution context. */ diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java index 4553845458541..2efa9220d2264 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; +import org.elasticsearch.client.HostSelector; + import static java.util.Collections.unmodifiableMap; /** @@ -35,6 +37,7 @@ public class ApiCallSection { private final Map params = new HashMap<>(); private final Map headers = new HashMap<>(); private final List> bodies = new ArrayList<>(); + private HostSelector hostSelector = HostSelector.ANY; public ApiCallSection(String api) { this.api = api; @@ -76,4 +79,18 @@ public void addBody(Map body) { public boolean hasBody() { return bodies.size() > 0; } + + /** + * Selects the node on which to run this request. + */ + public HostSelector getHostSelector() { + return hostSelector; + } + + /** + * Set the selector that decides which node can run this request. + */ + public void setHostSelector(HostSelector hostSelector) { + this.hostSelector = hostSelector; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index 0bcda1fc3f4bc..a15d498b106ab 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -92,10 +92,10 @@ public void addExecutableSection(ExecutableSection executableSection) { + "runners that do not support the [warnings] section can skip the test at line [" + doSection.getLocation().lineNumber + "]"); } - if (HostSelector.ANY != doSection.getHostSelector() + if (HostSelector.ANY != doSection.getApiCallSection().getHostSelector() && false == skipSection.getFeatures().contains("host_selector")) { - throw new IllegalArgumentException("Attempted to add a [do] with a [host] section without a corresponding [skip] so " - + "runners that do not support the [host_selector] section can skip the test at line [" + throw new IllegalArgumentException("Attempted to add a [do] with a [host_selector] section without a corresponding " + + "[skip] so runners that do not support the [host_selector] section can skip the test at line [" + doSection.getLocation().lineNumber + "]"); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 6285d3bcfb690..27054f25e8dad 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -88,6 +88,7 @@ public static DoSection parse(XContentParser parser) throws IOException { DoSection doSection = new DoSection(parser.getTokenLocation()); ApiCallSection apiCallSection = null; + HostSelector hostSelector = HostSelector.ANY; Map headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); List expectedWarnings = new ArrayList<>(); @@ -125,20 +126,20 @@ public static DoSection parse(XContentParser parser) throws IOException { } } } else if ("host_selector".equals(currentFieldName)) { + String selectorName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - String selectorName = null; if (token == XContentParser.Token.FIELD_NAME) { selectorName = parser.currentName(); } else if (token.isValue()) { - HostSelector original = doSection.getHostSelector(); - HostSelector newSelector = buildHostSelector( + final HostSelector original = hostSelector; + final HostSelector newSelector = buildHostSelector( parser.getTokenLocation(), selectorName, parser.text()); - doSection.setHostSelector(new HostSelector() { + hostSelector = new HostSelector() { @Override public boolean select(HttpHost host, HostMetadata meta) { return original.select(host, meta) && newSelector.select(host, meta); } - }); + }; } } } else if (currentFieldName != null) { // must be part of API call then @@ -173,6 +174,7 @@ public boolean select(HttpHost host, HostMetadata meta) { throw new IllegalArgumentException("client call section is mandatory within a do section"); } apiCallSection.addHeaders(headers); + apiCallSection.setHostSelector(hostSelector); doSection.setApiCallSection(apiCallSection); doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings)); } finally { @@ -187,7 +189,6 @@ public boolean select(HttpHost host, HostMetadata meta) { private String catchParam; private ApiCallSection apiCallSection; private List expectedWarningHeaders = emptyList(); - private HostSelector hostSelector = HostSelector.ANY; public DoSection(XContentLocation location) { this.location = location; @@ -226,20 +227,6 @@ public void setExpectedWarningHeaders(List expectedWarningHeaders) { this.expectedWarningHeaders = expectedWarningHeaders; } - /** - * Selects the node on which to run this request. - */ - public HostSelector getHostSelector() { - return hostSelector; - } - - /** - * Set the selector that decides which node can run this request. - */ - public void setHostSelector(HostSelector hostSelector) { - this.hostSelector = hostSelector; - } - @Override public XContentLocation getLocation() { return location; @@ -257,7 +244,7 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx try { ClientYamlTestResponse response = executionContext.callApi(apiCallSection.getApi(), apiCallSection.getParams(), - apiCallSection.getBodies(), apiCallSection.getHeaders()); + apiCallSection.getBodies(), apiCallSection.getHeaders(), apiCallSection.getHostSelector()); if (Strings.hasLength(catchParam)) { String catchStatusCode; if (catches.containsKey(catchParam)) { @@ -381,9 +368,8 @@ private static HostSelector buildHostSelector(XContentLocation location, String return new HostSelector() { @Override public boolean select(HttpHost host, HostMetadata meta) { - // NOCOMMIT actually compare the version - Version current = Version.CURRENT; - return current.onOrAfter(range[0]) && current.onOrBefore(range[1]); + Version version = Version.fromString(meta.version()); + return version.onOrAfter(range[0]) && version.onOrBefore(range[1]); } }; default: diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java index 2150baf59eab0..c818dec18e51d 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java @@ -20,24 +20,28 @@ package org.elasticsearch.test.rest.yaml; import org.apache.http.HttpEntity; +import org.elasticsearch.client.HostSelector; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + public class ClientYamlTestExecutionContextTests extends ESTestCase { public void testHeadersSupportStashedValueReplacement() throws IOException { final AtomicReference> headersRef = new AtomicReference<>(); final ClientYamlTestExecutionContext context = - new ClientYamlTestExecutionContext(null, randomBoolean()) { + new ClientYamlTestExecutionContext(null, () -> {}, randomBoolean()) { @Override ClientYamlTestResponse callApiInternal(String apiName, Map params, - HttpEntity entity, - Map headers) { + HttpEntity entity, Map headers, HostSelector hostSelector) { headersRef.set(headers); return null; } @@ -57,4 +61,20 @@ ClientYamlTestResponse callApiInternal(String apiName, Map param assertEquals("foo2", headersRef.get().get("foo")); assertEquals("baz bar1", headersRef.get().get("foo1")); } + + public void testNonDefaultHostSelectorSetsHostMetadata() throws IOException { + AtomicBoolean setHostMetadata = new AtomicBoolean(false); + final ClientYamlTestExecutionContext context = + new ClientYamlTestExecutionContext(null, () -> setHostMetadata.set(true), randomBoolean()) { + @Override + ClientYamlTestResponse callApiInternal(String apiName, Map params, + HttpEntity entity, Map headers, HostSelector hostSelector) { + return null; + } + }; + context.callApi(randomAlphaOfLength(2), emptyMap(), emptyList(), emptyMap(), HostSelector.ANY); + assertFalse(setHostMetadata.get()); + context.callApi(randomAlphaOfLength(2), emptyMap(), emptyList(), emptyMap(), HostSelector.NOT_MASTER); + assertTrue(setHostMetadata.get()); + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java index ecee131c7a28e..021c169510b1e 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.Version; +import org.elasticsearch.client.HostSelector; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; @@ -35,11 +36,12 @@ import static org.hamcrest.Matchers.nullValue; public class ClientYamlTestSectionTests extends AbstractClientYamlTestFragmentParserTestCase { - public void testAddingDoWithoutWarningWithoutSkip() { + public void testAddingDoWithoutSkips() { int lineNumber = between(1, 10000); ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); section.setSkipSection(SkipSection.EMPTY); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); + doSection.setApiCallSection(new ApiCallSection("test")); section.addExecutableSection(doSection); } @@ -49,6 +51,7 @@ public void testAddingDoWithWarningWithSkip() { section.setSkipSection(new SkipSection(null, singletonList("warnings"), null)); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setExpectedWarningHeaders(singletonList("foo")); + doSection.setApiCallSection(new ApiCallSection("test")); section.addExecutableSection(doSection); } @@ -58,11 +61,37 @@ public void testAddingDoWithWarningWithSkipButNotWarnings() { section.setSkipSection(new SkipSection(null, singletonList("yaml"), null)); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); doSection.setExpectedWarningHeaders(singletonList("foo")); + doSection.setApiCallSection(new ApiCallSection("test")); Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection)); assertEquals("Attempted to add a [do] with a [warnings] section without a corresponding [skip] so runners that do not support the" + " [warnings] section can skip the test at line [" + lineNumber + "]", e.getMessage()); } + public void testAddingDoWithHostSelectorWithSkip() { + int lineNumber = between(1, 10000); + ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); + section.setSkipSection(new SkipSection(null, singletonList("host_selector"), null)); + DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); + ApiCallSection apiCall = new ApiCallSection("test"); + apiCall.setHostSelector(HostSelector.NOT_MASTER); + doSection.setApiCallSection(apiCall); + section.addExecutableSection(doSection); + } + + public void testAddingDoWithHostSelectorWithSkipButNotWarnings() { + int lineNumber = between(1, 10000); + ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); + section.setSkipSection(new SkipSection(null, singletonList("yaml"), null)); + DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); + ApiCallSection apiCall = new ApiCallSection("test"); + apiCall.setHostSelector(HostSelector.NOT_MASTER); + doSection.setApiCallSection(apiCall); + Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection)); + assertEquals("Attempted to add a [do] with a [host_selector] section without a corresponding" + + " [skip] so runners that do not support the [host_selector] section can skip the test at" + + " line [" + lineNumber + "]", e.getMessage()); + } + public void testWrongIndentation() throws Exception { { XContentParser parser = createParser(YamlXContent.yamlXContent, diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java index 982eac4b80274..fe97fecf9985e 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java @@ -19,11 +19,16 @@ package org.elasticsearch.test.rest.yaml.section; +import org.apache.http.HttpHost; +import org.elasticsearch.client.HostMetadata; +import org.elasticsearch.client.HostSelector; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.yaml.YamlXContent; +import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse; import org.hamcrest.MatcherAssert; import java.io.IOException; @@ -31,11 +36,16 @@ import java.util.Map; import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class DoSectionTests extends AbstractClientYamlTestFragmentParserTestCase { @@ -496,7 +506,33 @@ public void testParseDoSectionExpectedWarnings() throws Exception { assertThat(doSection.getApiCallSection(), notNullValue()); assertThat(doSection.getExpectedWarningHeaders(), equalTo(singletonList( "just one entry this time"))); + } + + public void testHostSelector() throws IOException { + parser = createParser(YamlXContent.yamlXContent, + "host_selector:\n" + + " version: 5.2.0-6.0.0\n" + + "indices.get_field_mapping:\n" + + " index: test_index" + ); + DoSection doSection = DoSection.parse(parser); + HttpHost host = new HttpHost("dummy"); + HostMetadata.Roles roles = new HostMetadata.Roles(true, true, true); + assertNotSame(HostSelector.ANY, doSection.getApiCallSection().getHostSelector()); + assertTrue(doSection.getApiCallSection().getHostSelector() + .select(host, new HostMetadata("5.2.1", roles))); + assertFalse(doSection.getApiCallSection().getHostSelector() + .select(host, new HostMetadata("6.1.2", roles))); + assertFalse(doSection.getApiCallSection().getHostSelector() + .select(host, new HostMetadata("1.7.0", roles))); + ClientYamlTestExecutionContext context = mock(ClientYamlTestExecutionContext.class); + ClientYamlTestResponse mockResponse = mock(ClientYamlTestResponse.class); + when(context.callApi("indices.get_field_mapping", singletonMap("index", "test_index"), + emptyList(), emptyMap(), doSection.getApiCallSection().getHostSelector())).thenReturn(mockResponse); + doSection.execute(context); + verify(context).callApi("indices.get_field_mapping", singletonMap("index", "test_index"), + emptyList(), emptyMap(), doSection.getApiCallSection().getHostSelector()); } private void assertJsonEquals(Map actual, String expected) throws IOException { From 63d9514355c5baf284ad4311ef52130de27920cc Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 21 Mar 2018 20:18:33 -0400 Subject: [PATCH 04/27] Handle bound addresses --- .../elasticsearch/client/HostSelector.java | 10 +++ .../sniff/ElasticsearchHostsSniffer.java | 40 ++++++++---- .../client/sniff/HostsSniffer.java | 2 +- .../elasticsearch/client/sniff/Sniffer.java | 11 ++-- .../client/sniff/SnifferResult.java | 61 +++++++++++++++++++ .../test/rest/ESRestTestCase.java | 2 +- .../rest/yaml/ESClientYamlSuiteTestCase.java | 4 +- .../test/rest/yaml/section/DoSection.java | 13 ++++ 8 files changed, 122 insertions(+), 21 deletions(-) create mode 100644 client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java diff --git a/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java b/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java index f39f352f3dd35..88174d65c9fa1 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java +++ b/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java @@ -30,6 +30,11 @@ public interface HostSelector { public boolean select(HttpHost host, HostMetadata meta) { return true; } + + @Override + public String toString() { + return "ANY"; + } }; /** @@ -41,6 +46,11 @@ public boolean select(HttpHost host, HostMetadata meta) { public boolean select(HttpHost host, HostMetadata meta) { return meta != null && false == meta.roles().master(); } + + @Override + public String toString() { + return "NOT_MASTER"; + } }; /** diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java index e557b738c2c7b..8588a4b9f8937 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java @@ -34,8 +34,10 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -91,18 +93,19 @@ public ElasticsearchHostsSniffer(RestClient restClient, long sniffRequestTimeout * Calls the elasticsearch nodes info api, parses the response and returns all the found http hosts */ @Override - public Map sniffHosts() throws IOException { + public SnifferResult sniffHosts() throws IOException { Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams); return readHosts(response.getEntity()); } - private Map readHosts(HttpEntity entity) throws IOException { + private SnifferResult readHosts(HttpEntity entity) throws IOException { try (InputStream inputStream = entity.getContent()) { JsonParser parser = jsonFactory.createParser(inputStream); if (parser.nextToken() != JsonToken.START_OBJECT) { throw new IOException("expected data to start with an object"); } - Map hosts = new HashMap<>(); + List hosts = new ArrayList<>(); + Map hostMetadata = new HashMap<>(); while (parser.nextToken() != JsonToken.END_OBJECT) { if (parser.getCurrentToken() == JsonToken.START_OBJECT) { if ("nodes".equals(parser.getCurrentName())) { @@ -110,20 +113,22 @@ private Map readHosts(HttpEntity entity) throws IOExcept JsonToken token = parser.nextToken(); assert token == JsonToken.START_OBJECT; String nodeId = parser.getCurrentName(); - readHost(nodeId, parser, scheme, hosts); + readHost(nodeId, parser, scheme, hosts, hostMetadata); } } else { parser.skipChildren(); } } } - return hosts; + return new SnifferResult(hosts, hostMetadata); } } - private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Map hosts) throws IOException { + private static void readHost(String nodeId, JsonParser parser, Scheme scheme, List hosts, + Map hostMetadata) throws IOException { // NOCOMMIT test me against 2.x and 5.x - HttpHost httpHost = null; + HttpHost publishedHost = null; + List boundHosts = new ArrayList<>(); String fieldName = null; String version = null; boolean sawRoles = false; @@ -137,9 +142,16 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Ma if ("http".equals(fieldName)) { while (parser.nextToken() != JsonToken.END_OBJECT) { if (parser.getCurrentToken() == JsonToken.VALUE_STRING && "publish_address".equals(parser.getCurrentName())) { - URI boundAddressAsURI = URI.create(scheme + "://" + parser.getValueAsString()); - httpHost = new HttpHost(boundAddressAsURI.getHost(), boundAddressAsURI.getPort(), - boundAddressAsURI.getScheme()); + URI publishAddressAsURI = URI.create(scheme + "://" + parser.getValueAsString()); + publishedHost = new HttpHost(publishAddressAsURI.getHost(), publishAddressAsURI.getPort(), + publishAddressAsURI.getScheme()); + } else if (parser.currentToken() == JsonToken.START_ARRAY && "bound_address".equals(parser.getCurrentName())) { + while (parser.nextToken() != JsonToken.END_ARRAY) { + URI boundAddressAsURI = URI.create(scheme + "://" + parser.getValueAsString()); + boundHosts.add(new HttpHost(boundAddressAsURI.getHost(), boundAddressAsURI.getPort(), + boundAddressAsURI.getScheme())); + } + // NOCOMMIT test me } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) { parser.skipChildren(); } @@ -175,12 +187,16 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Ma } } //http section is not present if http is not enabled on the node, ignore such nodes - if (httpHost == null) { + if (publishedHost == null) { logger.debug("skipping node [" + nodeId + "] with http disabled"); } else { logger.trace("adding node [" + nodeId + "]"); assert sawRoles : "didn't see roles for [" + nodeId + "]"; - hosts.put(httpHost, new HostMetadata(version, new Roles(master, data, ingest))); + hosts.add(publishedHost); + HostMetadata meta = new HostMetadata(version, new Roles(master, data, ingest)); + for (HttpHost bound: boundHosts) { + hostMetadata.put(bound, meta); + } } } diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java index 95902169ca970..e27ef1dd5f8b0 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java @@ -33,5 +33,5 @@ public interface HostsSniffer { * Returns a {@link Map} from sniffed {@link HttpHost} to metadata * sniffed about the host. */ - Map sniffHosts() throws IOException; + SnifferResult sniffHosts() throws IOException; } diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java index 9e68071436d4b..37960d99454e1 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java @@ -120,21 +120,22 @@ void sniffOnFailure(HttpHost failedHost) { void sniff(HttpHost excludeHost, long nextSniffDelayMillis) { if (running.compareAndSet(false, true)) { try { - final Map sniffedHosts = hostsSniffer.sniffHosts(); + final SnifferResult sniffedHosts = hostsSniffer.sniffHosts(); logger.debug("sniffed hosts: " + sniffedHosts); if (excludeHost != null) { - sniffedHosts.remove(excludeHost); + sniffedHosts.hosts().remove(excludeHost); + sniffedHosts.hostMetadata().remove(excludeHost); } - if (sniffedHosts.isEmpty()) { + if (sniffedHosts.hosts().isEmpty()) { logger.warn("no hosts to set, hosts will be updated at the next sniffing round"); } else { HostMetadataResolver resolver = new HostMetadataResolver() { @Override public HostMetadata resolveMetadata(HttpHost host) { - return sniffedHosts.get(host); + return sniffedHosts.hostMetadata().get(host); } }; - this.restClient.setHosts(sniffedHosts.keySet(), resolver); + this.restClient.setHosts(sniffedHosts.hosts(), resolver); } } catch (Exception e) { logger.error("error while sniffing nodes", e); diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java new file mode 100644 index 0000000000000..0584b25d63a96 --- /dev/null +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.sniff; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.HostMetadata; + +import java.util.List; +import java.util.Map; + +/** + * Result of sniffing hosts. + */ +public final class SnifferResult { + /** + * List of all nodes in the cluster. + */ + private final List hosts; + /** + * Map from each address that each node is listening on to metadata about + * the node. + */ + private final Map hostMetadata; + + public SnifferResult(List hosts, Map hostMetadata) { + this.hosts = hosts; + this.hostMetadata = hostMetadata; + } + + /** + * List of all nodes in the cluster. + */ + public List hosts() { + return hosts; + } + + /** + * Map from each address that each node is listening on to metadata about + * the node. + */ + public Map hostMetadata() { + return hostMetadata; + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 27c4fe806fc1b..3ea3f1daf9573 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -564,7 +564,7 @@ protected void sniffHostMetadata(RestClient client) throws IOException { } ElasticsearchHostsSniffer sniffer = new ElasticsearchHostsSniffer( adminClient, ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT, scheme); - Map meta = sniffer.sniffHosts(); + Map meta = sniffer.sniffHosts().hostMetadata(); client.setHosts(clusterHosts, meta::get); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index dfc37d69466e7..49b3ab4fdaf27 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -121,9 +121,9 @@ public void initAndResetContext() throws Exception { } ClientYamlTestClient clientYamlTestClient = initClientYamlTestClient(restSpec, restClient, hosts, esVersion); restTestExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient, - () -> sniffHostMetadata(adminClient()), randomizeContentType()); + () -> sniffHostMetadata(client()), randomizeContentType()); adminExecutionContext = new ClientYamlTestExecutionContext(clientYamlTestClient, - () -> sniffHostMetadata(client()), false); + () -> sniffHostMetadata(adminClient()), false); String[] blacklist = resolvePathsProperty(REST_TESTS_BLACKLIST, null); blacklistPathMatchers = new ArrayList<>(); for (String entry : blacklist) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 27054f25e8dad..b163cd79a948f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -139,6 +139,11 @@ public static DoSection parse(XContentParser parser) throws IOException { public boolean select(HttpHost host, HostMetadata meta) { return original.select(host, meta) && newSelector.select(host, meta); } + + @Override + public String toString() { + return original + " AND " + newSelector; + } }; } } @@ -368,9 +373,17 @@ private static HostSelector buildHostSelector(XContentLocation location, String return new HostSelector() { @Override public boolean select(HttpHost host, HostMetadata meta) { + if (meta == null) { + throw new IllegalStateException("expected HostMetadata to be loaded!"); + } Version version = Version.fromString(meta.version()); return version.onOrAfter(range[0]) && version.onOrBefore(range[1]); } + + @Override + public String toString() { + return "version between [" + range[0] + "] and [" + range[1] + "]"; + } }; default: throw new IllegalArgumentException("unknown host_selector [" + name + "]"); From 17f92fe93621f0815fe4144ec10a2271b0c3ead8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 21 Mar 2018 20:18:57 -0400 Subject: [PATCH 05/27] Fix tests --- .../sniff/ElasticsearchHostsSniffer.java | 2 - .../client/sniff/SnifferResult.java | 20 +++++++++- .../sniff/ElasticsearchHostsSnifferTests.java | 40 +++++++++++-------- .../client/sniff/MockHostsSniffer.java | 8 ++-- .../documentation/SnifferDocumentation.java | 3 +- 5 files changed, 48 insertions(+), 25 deletions(-) diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java index 8588a4b9f8937..d3f964498590a 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java @@ -126,7 +126,6 @@ private SnifferResult readHosts(HttpEntity entity) throws IOException { private static void readHost(String nodeId, JsonParser parser, Scheme scheme, List hosts, Map hostMetadata) throws IOException { - // NOCOMMIT test me against 2.x and 5.x HttpHost publishedHost = null; List boundHosts = new ArrayList<>(); String fieldName = null; @@ -151,7 +150,6 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Li boundHosts.add(new HttpHost(boundAddressAsURI.getHost(), boundAddressAsURI.getPort(), boundAddressAsURI.getScheme())); } - // NOCOMMIT test me } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) { parser.skipChildren(); } diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java index 0584b25d63a96..527ec1fd32152 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; +import java.util.Objects; /** * Result of sniffing hosts. @@ -40,8 +41,8 @@ public final class SnifferResult { private final Map hostMetadata; public SnifferResult(List hosts, Map hostMetadata) { - this.hosts = hosts; - this.hostMetadata = hostMetadata; + this.hosts = Objects.requireNonNull(hosts, "hosts is required"); + this.hostMetadata = Objects.requireNonNull(hostMetadata, "hostMetadata is required"); } /** @@ -58,4 +59,19 @@ public List hosts() { public Map hostMetadata() { return hostMetadata; } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + SnifferResult other = (SnifferResult) obj; + return hosts.equals(other.hosts) + && hostMetadata.equals(other.hostMetadata); + } + + @Override + public int hashCode() { + return Objects.hash(hosts, hostMetadata); + } } diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java index 3f3237c2eb91e..74ee74499cafe 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java @@ -119,11 +119,11 @@ public void testSniffNodes() throws IOException { try (RestClient restClient = RestClient.builder(httpHost).build()) { ElasticsearchHostsSniffer sniffer = new ElasticsearchHostsSniffer(restClient, sniffRequestTimeout, scheme); try { - Map sniffedHosts = sniffer.sniffHosts(); + SnifferResult result = sniffer.sniffHosts(); if (sniffResponse.isFailure) { fail("sniffNodes should have failed"); } - assertEquals(sniffResponse.hosts, sniffedHosts); + assertEquals(sniffResponse.result, result); } catch(ResponseException e) { Response response = e.getResponse(); if (sniffResponse.isFailure) { @@ -178,7 +178,8 @@ public void handle(HttpExchange httpExchange) throws IOException { private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme scheme) throws IOException { int numNodes = RandomNumbers.randomIntBetween(getRandom(), 1, 5); - Map hosts = new HashMap<>(numNodes); + List hosts = new ArrayList<>(numNodes); + Map hostMetadata = new HashMap<>(numNodes); JsonFactory jsonFactory = new JsonFactory(); StringWriter writer = new StringWriter(); JsonGenerator generator = jsonFactory.createGenerator(writer); @@ -212,15 +213,20 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme } boolean isHttpEnabled = rarely() == false; if (isHttpEnabled) { - hosts.put(httpHost, metadata); + hosts.add(httpHost); + hostMetadata.put(httpHost, metadata); generator.writeObjectFieldStart("http"); - if (getRandom().nextBoolean()) { - generator.writeArrayFieldStart("bound_address"); - generator.writeString("[fe80::1]:" + port); - generator.writeString("[::1]:" + port); - generator.writeString("127.0.0.1:" + port); - generator.writeEndArray(); + generator.writeArrayFieldStart("bound_address"); + generator.writeString(httpHost.toHostString()); + if (randomBoolean()) { + int extras = between(1, 5); + for (int e = 0; e < extras; e++) { + HttpHost extraHost = new HttpHost(httpHost.getHostName() + e, port, scheme.toString()); + hostMetadata.put(extraHost, metadata); + generator.writeString(extraHost.toHostString()); + } } + generator.writeEndArray(); if (getRandom().nextBoolean()) { generator.writeObjectFieldStart("bogus_object"); generator.writeEndObject(); @@ -270,18 +276,18 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme generator.writeEndObject(); generator.writeEndObject(); generator.close(); - return SniffResponse.buildResponse(writer.toString(), hosts); + return SniffResponse.buildResponse(writer.toString(), new SnifferResult(hosts, hostMetadata)); } private static class SniffResponse { private final String nodesInfoBody; private final int nodesInfoResponseCode; - private final Map hosts; + private final SnifferResult result; private final boolean isFailure; - SniffResponse(String nodesInfoBody, Map hosts, boolean isFailure) { + SniffResponse(String nodesInfoBody, SnifferResult result, boolean isFailure) { this.nodesInfoBody = nodesInfoBody; - this.hosts = hosts; + this.result = result; this.isFailure = isFailure; if (isFailure) { this.nodesInfoResponseCode = randomErrorResponseCode(); @@ -291,11 +297,11 @@ private static class SniffResponse { } static SniffResponse buildFailure() { - return new SniffResponse("", Collections.emptyMap(), true); + return new SniffResponse("", null, true); } - static SniffResponse buildResponse(String nodesInfoBody, Map hosts) { - return new SniffResponse(nodesInfoBody, hosts, false); + static SniffResponse buildResponse(String nodesInfoBody, SnifferResult result) { + return new SniffResponse(nodesInfoBody, result, false); } } diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java index a14ddcb683b79..40aab7228a80b 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java @@ -31,8 +31,10 @@ */ class MockHostsSniffer implements HostsSniffer { @Override - public Map sniffHosts() throws IOException { - return Collections.singletonMap(new HttpHost("localhost", 9200), - new HostMetadata("mock version", new HostMetadata.Roles(false, false, false))); + public SnifferResult sniffHosts() throws IOException { + HttpHost host = new HttpHost("localhost", 9200); + return new SnifferResult(Collections.singletonList(host), + Collections.singletonMap(new HttpHost("localhost", 9200), + new HostMetadata("mock version", new HostMetadata.Roles(false, false, false)))); } } diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java index 0b541de7b8ebb..91699be46f5fd 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java @@ -26,6 +26,7 @@ import org.elasticsearch.client.sniff.HostsSniffer; import org.elasticsearch.client.sniff.SniffOnFailureListener; import org.elasticsearch.client.sniff.Sniffer; +import org.elasticsearch.client.sniff.SnifferResult; import java.io.IOException; import java.util.List; @@ -121,7 +122,7 @@ public void testUsage() throws IOException { .build(); HostsSniffer hostsSniffer = new HostsSniffer() { @Override - public Map sniffHosts() throws IOException { + public SnifferResult sniffHosts() throws IOException { return null; // <1> } }; From 3bb20cb536f2036dae1296c42ba66a914959eb3e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 22 Mar 2018 12:16:40 -0400 Subject: [PATCH 06/27] Tests for nextHost --- .../client/AbstractRestClientActions.java | 2 +- .../elasticsearch/client/DeadHostState.java | 5 + .../org/elasticsearch/client/RestClient.java | 180 +++++++++++++----- .../elasticsearch/client/RestClientView.java | 3 +- .../elasticsearch/client/RestClientTests.java | 143 ++++++++++++++ 5 files changed, 280 insertions(+), 53 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java index 8ee37ce25199c..cd398b3e54012 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java @@ -41,7 +41,7 @@ abstract class AbstractRestClientActions implements RestClientActions { abstract void performRequestAsyncNoCatch(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, Header[] headers); + ResponseListener responseListener, Header[] headers) throws IOException; /** * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response diff --git a/client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java b/client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java index a7b222da70e1d..1412fb589e620 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java +++ b/client/rest/src/main/java/org/elasticsearch/client/DeadHostState.java @@ -53,6 +53,11 @@ private DeadHostState() { this.failedAttempts = previousDeadHostState.failedAttempts + 1; } + DeadHostState(int failedAttempts, long deadUntilNanos) { + this.failedAttempts = failedAttempts; + this.deadUntilNanos = deadUntilNanos; + } + /** * Returns the timestamp (nanos) till the host is supposed to stay dead without being retried. * After that the host should be retried. diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index be1ecf6389b2f..0f94de6df98a4 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -87,9 +87,14 @@ * Requests can be traced by enabling trace logging for "tracer". The trace logger outputs requests and responses in curl format. */ public class RestClient extends AbstractRestClientActions implements Closeable { - private static final Log logger = LogFactory.getLog(RestClient.class); + /** + * The maximum number of attempts that {@link #nextHost(HostSelector)} makes + * before giving up and failing the request. + */ + private static final int MAX_NEXT_HOSTS_ATTEMPTS = 10; + private final CloseableHttpAsyncClient client; // We don't rely on default headers supported by HttpAsyncClient as those cannot be replaced. // These are package private for tests. @@ -174,7 +179,7 @@ public RestClientActions withHostSelector(HostSelector hostSelector) { @Override final void performRequestAsyncNoCatch(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, Header[] headers) { + ResponseListener responseListener, Header[] headers) throws IOException { // Requests made directly to the client use the noop HostSelector. HostSelector hostSelector = HostSelector.ANY; performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, @@ -182,8 +187,8 @@ final void performRequestAsyncNoCatch(String method, String endpoint, Map params, - HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, HostSelector hostSelector, Header[] headers) { + HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, + ResponseListener responseListener, HostSelector hostSelector, Header[] headers) throws IOException { Objects.requireNonNull(params, "params must not be null"); Map requestParams = new HashMap<>(params); //ignore is a special parameter supported by the clients, shouldn't be sent to es @@ -312,60 +317,133 @@ private void setHeaders(HttpRequest httpRequest, Header[] requestHeaders) { } /** - * Returns an {@link Iterable} of hosts to be used for a request call. - * Ideally, the first host is retrieved from the iterable and used successfully for the request. - * Otherwise, after each failure the next host has to be retrieved from the iterator so that the request can be retried until - * there are no more hosts available to retry against. The maximum total of attempts is equal to the number of hosts in the iterable. - * The iterator returned will never be empty. In case there are no healthy hosts available, or dead ones to be be retried, - * one dead host gets returned so that it can be retried. + * Returns a non-empty {@link Iterator} of hosts to be used for a request + * that match the {@link HostSelector}. + *

+ * If there are no living hosts that match the {@link HostSelector} + * this will return the dead host that matches the {@link HostSelector} + * that is closest to being revived. + *

+ * If no living and no dead hosts match the selector we retry a few + * times to handle concurrent modifications of the list of dead hosts. + * We never block the thread or {@link Thread#sleep} or anything like + * that. If the retries fail this throws a {@link IOException}. + * @throws IOException if no hosts are available */ - private HostTuple> nextHost(HostSelector hostSelector) { - final HostTuple> hostTuple = this.hostTuple; - Collection nextHosts = Collections.emptySet(); + private HostTuple> nextHost(HostSelector hostSelector) throws IOException { + int attempts = 0; + NextHostsResult result; + /* + * Try to fetch the hosts to which we can send the request. It is possible that + * this returns an empty collection because of concurrent modification to the + * blacklist. + */ do { - Set filteredHosts = new HashSet<>(hostTuple.hosts); - for (Iterator hostItr = filteredHosts.iterator(); hostItr.hasNext();) { - final HttpHost host = hostItr.next(); - if (false == hostSelector.select(host, hostTuple.metaResolver.resolveMetadata(host))) { - hostItr.remove(); + final HostTuple> hostTuple = this.hostTuple; + result = nextHostsOneTime(hostTuple, blacklist, lastHostIndex, System.nanoTime(), hostSelector); + if (result.hosts == null) { + if (logger.isDebugEnabled()) { + logger.debug("No hosts avialable. Will retry. Failure is " + result.describeFailure()); } + } else { + // Success! + return new HostTuple<>(result.hosts.iterator(), hostTuple.authCache, hostTuple.metaResolver); } - for (Map.Entry entry : blacklist.entrySet()) { - if (System.nanoTime() - entry.getValue().getDeadUntilNanos() < 0) { - filteredHosts.remove(entry.getKey()); - } + attempts++; + } while (attempts < MAX_NEXT_HOSTS_ATTEMPTS); + throw new IOException("No hosts available for request. Last failure was " + result.describeFailure()); + } + + static class NextHostsResult { + /** + * Number of hosts filtered from the list because they are + * dead. + */ + int blacklisted = 0; + /** + * Number of hosts filtered from the list because the. + * {@link HostSelector} didn't approve of them. + */ + int selectorRejected = 0; + /** + * Number of hosts that could not be revived because the + * {@link HostSelector} didn't approve of them. + */ + int selectorBlockedRevival = 0; + /** + * {@code null} if we failed to find any hosts, a list of + * hosts to use if we found any. + */ + Collection hosts = null; + + public String describeFailure() { + assert hosts == null : "describeFailure shouldn't be called with successful request"; + return "[blacklisted=" + blacklisted + + ", selectorRejected=" + selectorRejected + + ", selectorBlockedRevival=" + selectorBlockedRevival + "]]"; + } + } + static NextHostsResult nextHostsOneTime(HostTuple> hostTuple, + Map blacklist, AtomicInteger lastHostIndex, + long now, HostSelector hostSelector) { + NextHostsResult result = new NextHostsResult(); + Set filteredHosts = new HashSet<>(hostTuple.hosts); + for (Map.Entry entry : blacklist.entrySet()) { + if (now - entry.getValue().getDeadUntilNanos() < 0) { + filteredHosts.remove(entry.getKey()); + result.blacklisted++; } - if (filteredHosts.isEmpty()) { - /* - * Last resort: If there are no good host to use, return a single dead one, - * the one that's closest to being retried *and* matches the selector. - */ - List> sortedHosts = new ArrayList<>(blacklist.entrySet()); - if (sortedHosts.size() > 0) { - Collections.sort(sortedHosts, new Comparator>() { - @Override - public int compare(Map.Entry o1, Map.Entry o2) { - return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos()); - } - }); - Iterator> nodeItr = sortedHosts.iterator(); - while (nodeItr.hasNext()) { - final HttpHost deadHost = nodeItr.next().getKey(); - if (hostSelector.select(deadHost, hostTuple.metaResolver.resolveMetadata(deadHost))) { - logger.trace("resurrecting host [" + deadHost + "]"); - nextHosts = Collections.singleton(deadHost); - break; - } - } + } + for (Iterator hostItr = filteredHosts.iterator(); hostItr.hasNext();) { + final HttpHost host = hostItr.next(); + if (false == hostSelector.select(host, hostTuple.metaResolver.resolveMetadata(host))) { + hostItr.remove(); + result.selectorRejected++; + } + } + if (false == filteredHosts.isEmpty()) { + /* + * Normal case: we have at least one non-dead host that the hostSelector + * is fine with. Rotate the list so repeated requests with the same blacklist + * and the same selector round robin. If you use a different HostSelector + * or a host goes dark then the round robin won't be perfect but that should + * be fine. + */ + List rotatedHosts = new ArrayList<>(filteredHosts); + int i = lastHostIndex.getAndIncrement(); + Collections.rotate(rotatedHosts, i); + result.hosts = rotatedHosts; + return result; + } + /* + * Last resort: If there are no good hosts to use, return a single dead one, + * the one that's closest to being retried *and* matches the selector. + */ + List> sortedHosts = new ArrayList<>(blacklist.entrySet()); + if (sortedHosts.isEmpty()) { + // There are no dead hosts to revive. Return a failed result and we'll retry. + return result; + } + Collections.sort(sortedHosts, new Comparator>() { + @Override + public int compare(Map.Entry o1, Map.Entry o2) { + return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos()); + } + }); + Iterator> nodeItr = sortedHosts.iterator(); + while (nodeItr.hasNext()) { + final HttpHost deadHost = nodeItr.next().getKey(); + if (hostSelector.select(deadHost, hostTuple.metaResolver.resolveMetadata(deadHost))) { + if (logger.isTraceEnabled()) { + logger.trace("resurrecting host [" + deadHost + "]"); } - // NOCOMMIT we get here if the selector rejects all hosts + result.hosts = Collections.singleton(deadHost); + return result; } else { - List rotatedHosts = new ArrayList<>(filteredHosts); - Collections.rotate(rotatedHosts, rotatedHosts.size() - lastHostIndex.getAndIncrement()); - nextHosts = rotatedHosts; + result.selectorBlockedRevival++; } - } while(nextHosts.isEmpty()); - return new HostTuple<>(nextHosts.iterator(), hostTuple.authCache, hostTuple.metaResolver); + } + return result; } /** @@ -537,7 +615,7 @@ public void onFailure(HttpHost host) { * {@code HostTuple} enables the {@linkplain HttpHost}s and {@linkplain AuthCache} to be set together in a thread * safe, volatile way. */ - private static class HostTuple { + static class HostTuple { final T hosts; final AuthCache authCache; final HostMetadataResolver metaResolver; diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java index 9838a14a46feb..616429dcec061 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java @@ -19,6 +19,7 @@ package org.elasticsearch.client; +import java.io.IOException; import java.util.Map; import org.apache.http.Header; @@ -57,7 +58,7 @@ public boolean select(HttpHost host, HostMetadata meta) { @Override final void performRequestAsyncNoCatch(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, Header[] headers) { + ResponseListener responseListener, Header[] headers) throws IOException { delegate.performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, responseListener, hostSelector, headers); } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java index 142a43bde140d..7648b3023d5e6 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java @@ -22,14 +22,26 @@ import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; +import org.elasticsearch.client.HostMetadata.HostMetadataResolver; +import org.elasticsearch.client.RestClient.HostTuple; +import org.elasticsearch.client.RestClient.NextHostsResult; import java.io.IOException; import java.net.URI; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -148,6 +160,137 @@ public void testBuildUriLeavesPathUntouched() { } } + public void testNextHostsOneTime() { + int iterations = 1000; + HttpHost h1 = new HttpHost("1"); + HttpHost h2 = new HttpHost("2"); + HttpHost h3 = new HttpHost("3"); + Set hosts = new HashSet<>(); + hosts.add(h1); + hosts.add(h2); + hosts.add(h3); + + HostMetadataResolver versionIsName = new HostMetadataResolver() { + @Override + public HostMetadata resolveMetadata(HttpHost host) { + return new HostMetadata(host.toHostString(), new HostMetadata.Roles(true, true, true)); + } + }; + HostSelector not1 = new HostSelector() { + @Override + public boolean select(HttpHost host, HostMetadata meta) { + return false == "1".equals(meta.version()); + } + }; + HostSelector noHosts = new HostSelector() { + @Override + public boolean select(HttpHost host, HostMetadata meta) { + return false; + } + }; + + HostTuple> hostTuple = new HostTuple<>(hosts, null, versionIsName); + Map blacklist = new HashMap<>(); + AtomicInteger lastHostIndex = new AtomicInteger(0); + long now = 0; + + // Normal case + NextHostsResult result = RestClient.nextHostsOneTime(hostTuple, blacklist, + lastHostIndex, now, HostSelector.ANY); + assertThat(result.hosts, containsInAnyOrder(h1, h2, h3)); + List expectedHosts = new ArrayList<>(result.hosts); + // Calling it again rotates the set of results + for (int i = 0; i < iterations; i++) { + Collections.rotate(expectedHosts, 1); + assertEquals(expectedHosts, RestClient.nextHostsOneTime(hostTuple, blacklist, + lastHostIndex, now, HostSelector.ANY).hosts); + } + + // Exclude some host + lastHostIndex.set(0); + result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, now, not1); + assertThat(result.hosts, containsInAnyOrder(h2, h3)); // h1 excluded + assertEquals(0, result.blacklisted); + assertEquals(1, result.selectorRejected); + assertEquals(0, result.selectorBlockedRevival); + expectedHosts = new ArrayList<>(result.hosts); + // Calling it again rotates the set of results + for (int i = 0; i < iterations; i++) { + Collections.rotate(expectedHosts, 1); + assertEquals(expectedHosts, RestClient.nextHostsOneTime(hostTuple, blacklist, + lastHostIndex, now, not1).hosts); + } + + /* + * Try a HostSelector that excludes all hosts. This should + * return a failure. + */ + result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, now, noHosts); + assertNull(result.hosts); + assertEquals(0, result.blacklisted); + assertEquals(3, result.selectorRejected); + assertEquals(0, result.selectorBlockedRevival); + + /* + * Mark all hosts as dead and look up at a time *after* the + * revival time. This should return all hosts. + */ + blacklist.put(h1, new DeadHostState(1, 1)); + blacklist.put(h2, new DeadHostState(1, 2)); + blacklist.put(h3, new DeadHostState(1, 3)); + lastHostIndex.set(0); + now = 1000; + result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, + now, HostSelector.ANY); + assertThat(result.hosts, containsInAnyOrder(h1, h2, h3)); + assertEquals(0, result.blacklisted); + assertEquals(0, result.selectorRejected); + assertEquals(0, result.selectorBlockedRevival); + expectedHosts = new ArrayList<>(result.hosts); + // Calling it again rotates the set of results + for (int i = 0; i < iterations; i++) { + Collections.rotate(expectedHosts, 1); + assertEquals(expectedHosts, RestClient.nextHostsOneTime(hostTuple, blacklist, + lastHostIndex, now, HostSelector.ANY).hosts); + } + + /* + * Now try with the hosts dead and *not* past their dead time. + * Only the host closest to revival should come back. + */ + now = 0; + result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, + now, HostSelector.ANY); + assertEquals(Collections.singleton(h1), result.hosts); + assertEquals(3, result.blacklisted); + assertEquals(0, result.selectorRejected); + assertEquals(0, result.selectorBlockedRevival); + + /* + * Now try with the hosts dead and *not* past their dead time + * *and* a host selector that removes the host that is closest + * to being revived. The second closest host should come back. + */ + result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, + now, not1); + assertEquals(Collections.singleton(h2), result.hosts); + assertEquals(3, result.blacklisted); + assertEquals(0, result.selectorRejected); + assertEquals(1, result.selectorBlockedRevival); + + /* + * Try a HostSelector that excludes all hosts. This should + * return a failure, but a different failure than normal + * because it'll block revival rather than outright reject + * healthy hosts. + */ + result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, now, noHosts); + assertNull(result.hosts); + assertEquals(3, result.blacklisted); + assertEquals(0, result.selectorRejected); + assertEquals(3, result.selectorBlockedRevival); + } + private static RestClient createRestClient() { HttpHost[] hosts = new HttpHost[]{new HttpHost("localhost", 9200)}; return new RestClient(mock(CloseableHttpAsyncClient.class), randomLongBetween(1_000, 30_000), From c69d24a3ea0c8b6ff8b0bfdc8922621e7bfed361 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 22 Mar 2018 14:08:52 -0400 Subject: [PATCH 07/27] Revert changes to high level client --- .../client/RestHighLevelClient.java | 10 ++++------ .../client/CustomRestHighLevelClientTests.java | 6 +++--- .../client/RestHighLevelClientExtTests.java | 2 +- .../client/RestHighLevelClientTests.java | 16 ++++++---------- .../client/AbstractRestClientActions.java | 8 ++++---- .../org/elasticsearch/client/RestClient.java | 14 ++++++++++++++ 6 files changed, 32 insertions(+), 24 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index dbd94c8e0c41b..472c21f06517e 100755 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -181,9 +181,9 @@ */ public class RestHighLevelClient implements Closeable { - private final RestClientActions client; + private final RestClient client; private final NamedXContentRegistry registry; - private final CheckedConsumer doClose; + private final CheckedConsumer doClose; private final IndicesClient indicesClient = new IndicesClient(this); private final ClusterClient clusterClient = new ClusterClient(this); @@ -196,14 +196,12 @@ public RestHighLevelClient(RestClientBuilder restClientBuilder) { this(restClientBuilder, Collections.emptyList()); } - // NOCOMMIT revert so we can use the same wrapping pattern here? - /** * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the * {@link RestClient} to be used to perform requests and parsers for custom response sections added to Elasticsearch through plugins. */ protected RestHighLevelClient(RestClientBuilder restClientBuilder, List namedXContentEntries) { - this(restClientBuilder.build(), (RestClientActions c) -> ((RestClient) c).close(), namedXContentEntries); + this(restClientBuilder.build(), RestClient::close, namedXContentEntries); } /** @@ -213,7 +211,7 @@ protected RestHighLevelClient(RestClientBuilder restClientBuilder, List doClose, + protected RestHighLevelClient(RestClient restClient, CheckedConsumer doClose, List namedXContentEntries) { this.client = Objects.requireNonNull(restClient, "restClient must not be null"); this.doClose = Objects.requireNonNull(doClose, "doClose consumer must not be null"); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java index 5e201662c4fec..42496822090fd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CustomRestHighLevelClientTests.java @@ -76,7 +76,7 @@ public class CustomRestHighLevelClientTests extends ESTestCase { @Before public void initClients() throws IOException { if (restHighLevelClient == null) { - final RestClientActions restClient = mock(RestClientActions.class); + final RestClient restClient = mock(RestClient.class); restHighLevelClient = new CustomRestClient(restClient); doAnswer(mock -> mockPerformRequest((Header) mock.getArguments()[4])) @@ -172,8 +172,8 @@ private Response mockPerformRequest(Header httpHeader) throws IOException { */ static class CustomRestClient extends RestHighLevelClient { - private CustomRestClient(RestClientActions restClient) { - super(restClient, c -> {}, Collections.emptyList()); + private CustomRestClient(RestClient restClient) { + super(restClient, RestClient::close, Collections.emptyList()); } MainResponse custom(MainRequest mainRequest, Header... headers) throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java index 7903eb0a6a3ba..970ffb15f083b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientExtTests.java @@ -69,7 +69,7 @@ public void testParseEntityCustomResponseSection() throws IOException { private static class RestHighLevelClientExt extends RestHighLevelClient { private RestHighLevelClientExt(RestClient restClient) { - super(restClient, c -> restClient.close(), getNamedXContentsExt()); + super(restClient, RestClient::close, getNamedXContentsExt()); } private static List getNamedXContentsExt() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 184833c00452a..2dab6d12b6a0b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -113,26 +113,22 @@ public class RestHighLevelClientTests extends ESTestCase { private static final ProtocolVersion HTTP_PROTOCOL = new ProtocolVersion("http", 1, 1); private static final RequestLine REQUEST_LINE = new BasicRequestLine(HttpGet.METHOD_NAME, "/", HTTP_PROTOCOL); - private RestClientActions restClient; - private CheckedConsumer doClose; + private RestClient restClient; private RestHighLevelClient restHighLevelClient; @Before public void initClient() { - restClient = mock(RestClientActions.class); - @SuppressWarnings("unchecked") - CheckedConsumer doClose = mock(CheckedConsumer.class); - this.doClose = doClose; - restHighLevelClient = new RestHighLevelClient(restClient, doClose, Collections.emptyList()); + restClient = mock(RestClient.class); + restHighLevelClient = new RestHighLevelClient(restClient, RestClient::close, Collections.emptyList()); } public void testCloseIsIdempotent() throws IOException { restHighLevelClient.close(); - verify(doClose, times(1)).accept(restClient); + verify(restClient, times(1)).close(); restHighLevelClient.close(); - verify(doClose, times(2)).accept(restClient); + verify(restClient, times(2)).close(); restHighLevelClient.close(); - verify(doClose, times(3)).accept(restClient); + verify(restClient, times(3)).close(); } public void testPingSuccessful() throws IOException { diff --git a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java index cd398b3e54012..8e13eb9c3bc47 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java @@ -95,8 +95,8 @@ public final Response performRequest(String method, String endpoint, Map params, + @Override // TODO this method is not final so the tests for the High Level REST Client don't have to change. We'll fix this soon. + public Response performRequest(String method, String endpoint, Map params, HttpEntity entity, Header... headers) throws IOException { return performRequest(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, headers); } @@ -184,8 +184,8 @@ public final void performRequestAsync(String method, String endpoint, Map params, + @Override // TODO this method is not final so the tests for the High Level REST Client don't have to change. We'll fix this soon. + public void performRequestAsync(String method, String endpoint, Map params, HttpEntity entity, ResponseListener responseListener, Header... headers) { performRequestAsync(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, responseListener, headers); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 0f94de6df98a4..b7aa952d2674f 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -176,6 +176,20 @@ public RestClientActions withHostSelector(HostSelector hostSelector) { return new RestClientView(this, hostSelector); } + // TODO this exists entirely to so we don't have to change much in the high level rest client tests. We'll remove in a followup. + @Override + public Response performRequest(String method, String endpoint, Map params, + HttpEntity entity, Header... headers) throws IOException { + return super.performRequest(method, endpoint, params, entity, headers); + } + + // TODO this exists entirely to so we don't have to change much in the high level rest client tests. We'll remove in a followup. + @Override + public void performRequestAsync(String method, String endpoint, Map params, + HttpEntity entity, ResponseListener responseListener, Header... headers) { + super.performRequestAsync(method, endpoint, params, entity, responseListener, headers); + } + @Override final void performRequestAsyncNoCatch(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, From 74e4655ebc43ec2328c22ba9278cbc0ae1a116ca Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 22 Mar 2018 14:09:42 -0400 Subject: [PATCH 08/27] Revert another --- .../main/java/org/elasticsearch/client/RestHighLevelClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 472c21f06517e..bf80aa7720741 100755 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -223,7 +223,7 @@ protected RestHighLevelClient(RestClient restClient, CheckedConsumer Date: Thu, 22 Mar 2018 14:10:23 -0400 Subject: [PATCH 09/27] Another revert --- .../java/org/elasticsearch/client/RestHighLevelClientTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 2dab6d12b6a0b..b8315bd59fa43 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -54,7 +54,6 @@ import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.xcontent.NamedXContentRegistry; From 00dd77cfeb684c8ee54834d5d0796b72d6375c56 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 22 Mar 2018 14:34:37 -0400 Subject: [PATCH 10/27] Fix javadocs --- .../client/AbstractRestClientActions.java | 127 +----------------- .../elasticsearch/client/HostMetadata.java | 38 ++++-- .../elasticsearch/client/HostSelector.java | 17 ++- .../client/RestClientActions.java | 6 + .../elasticsearch/client/RestClientView.java | 10 +- 5 files changed, 58 insertions(+), 140 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java index 8e13eb9c3bc47..ac3ad38d2b5e6 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java @@ -36,6 +36,10 @@ import org.apache.http.HttpEntity; import org.apache.http.conn.ConnectTimeoutException; +/** + * Abstract implementation of {@link RestClientActions} shared by + * {@link RestClient} and {@link RestClientView}. + */ abstract class AbstractRestClientActions implements RestClientActions { abstract SyncResponseListener syncResponseListener(); @@ -43,92 +47,22 @@ abstract void performRequestAsyncNoCatch(String method, String endpoint, MapemptyMap(), null, headers); } - /** - * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response - * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, Header...)} but without request body. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param headers the optional request headers - * @return the response returned by Elasticsearch - * @throws IOException in case of a problem or the connection was aborted - * @throws ClientProtocolException in case of an http protocol error - * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error - */ @Override public final Response performRequest(String method, String endpoint, Map params, Header... headers) throws IOException { return performRequest(method, endpoint, params, (HttpEntity)null, headers); } - /** - * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response - * to be returned. Shortcut to {@link #performRequest(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, Header...)} - * which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance, - * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param entity the body of the request, null if not applicable - * @param headers the optional request headers - * @return the response returned by Elasticsearch - * @throws IOException in case of a problem or the connection was aborted - * @throws ClientProtocolException in case of an http protocol error - * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error - */ @Override // TODO this method is not final so the tests for the High Level REST Client don't have to change. We'll fix this soon. public Response performRequest(String method, String endpoint, Map params, HttpEntity entity, Header... headers) throws IOException { return performRequest(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, headers); } - /** - * Sends a request to the Elasticsearch cluster that the client points to. Blocks until the request is completed and returns - * its response or fails by throwing an exception. Selects a host out of the provided ones in a round-robin fashion. Failing hosts - * are marked dead and retried after a certain amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times - * they previously failed (the more failures, the later they will be retried). In case of failures all of the alive nodes (or dead - * nodes that deserve a retry) are retried until one responds or none of them does, in which case an {@link IOException} will be thrown. - * - * This method works by performing an asynchronous call and waiting - * for the result. If the asynchronous call throws an exception we wrap - * it and rethrow it so that the stack trace attached to the exception - * contains the call site. While we attempt to preserve the original - * exception this isn't always possible and likely haven't covered all of - * the cases. You can get the original exception from - * {@link Exception#getCause()}. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param entity the body of the request, null if not applicable - * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one - * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP - * connection on the client side. - * @param headers the optional request headers - * @return the response returned by Elasticsearch - * @throws IOException in case of a problem or the connection was aborted - * @throws ClientProtocolException in case of an http protocol error - * @throws ResponseException in case Elasticsearch responded with a status code that indicated an error - */ @Override public final Response performRequest(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, @@ -139,75 +73,22 @@ public final Response performRequest(String method, String endpoint, MapemptyMap(), null, responseListener, headers); } - /** - * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead - * the provided {@link ResponseListener} will be notified upon completion or failure. Shortcut to - * {@link #performRequestAsync(String, String, Map, HttpEntity, ResponseListener, Header...)} but without request body. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails - * @param headers the optional request headers - */ @Override public final void performRequestAsync(String method, String endpoint, Map params, ResponseListener responseListener, Header... headers) { performRequestAsync(method, endpoint, params, null, responseListener, headers); } - /** - * Sends a request to the Elasticsearch cluster that the client points to. Doesn't wait for the response, instead - * the provided {@link ResponseListener} will be notified upon completion or failure. - * Shortcut to {@link #performRequestAsync(String, String, Map, HttpEntity, HttpAsyncResponseConsumerFactory, ResponseListener, - * Header...)} which doesn't require specifying an {@link HttpAsyncResponseConsumerFactory} instance, - * {@link HttpAsyncResponseConsumerFactory} will be used to create the needed instances of {@link HttpAsyncResponseConsumer}. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param entity the body of the request, null if not applicable - * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails - * @param headers the optional request headers - */ @Override // TODO this method is not final so the tests for the High Level REST Client don't have to change. We'll fix this soon. public void performRequestAsync(String method, String endpoint, Map params, HttpEntity entity, ResponseListener responseListener, Header... headers) { performRequestAsync(method, endpoint, params, entity, HttpAsyncResponseConsumerFactory.DEFAULT, responseListener, headers); } - /** - * Sends a request to the Elasticsearch cluster that the client points to. The request is executed asynchronously - * and the provided {@link ResponseListener} gets notified upon request completion or failure. - * Selects a host out of the provided ones in a round-robin fashion. Failing hosts are marked dead and retried after a certain - * amount of time (minimum 1 minute, maximum 30 minutes), depending on how many times they previously failed (the more failures, - * the later they will be retried). In case of failures all of the alive nodes (or dead nodes that deserve a retry) are retried - * until one responds or none of them does, in which case an {@link IOException} will be thrown. - * - * @param method the http method - * @param endpoint the path of the request (without host and port) - * @param params the query_string parameters - * @param entity the body of the request, null if not applicable - * @param httpAsyncResponseConsumerFactory the {@link HttpAsyncResponseConsumerFactory} used to create one - * {@link HttpAsyncResponseConsumer} callback per retry. Controls how the response body gets streamed from a non-blocking HTTP - * connection on the client side. - * @param responseListener the {@link ResponseListener} to notify when the request is completed or fails - * @param headers the optional request headers - */ @Override public final void performRequestAsync(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, diff --git a/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java b/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java index 9f81c793df3f2..19e87b674d194 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java +++ b/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java @@ -23,27 +23,40 @@ import org.apache.http.HttpHost; +/** + * Metadata about an {@link HttpHost} running Elasticsearch. + */ public class HostMetadata { - public static final HostMetadataResolver EMPTY_RESOLVER = new HostMetadataResolver() { - @Override - public HostMetadata resolveMetadata(HttpHost host) { - return null; - } - }; /** - * Look up metadata about the provided host. Implementers should not make network - * calls, instead, they should look up previously fetched data. See Elasticsearch's - * Sniffer for an example implementation. + * Look up metadata about the provided host. Implementers should not + * make network calls, instead, they should look up previously fetched + * data. See Elasticsearch's Sniffer for an example implementation. */ public interface HostMetadataResolver { /** - * @return {@link HostMetadat} about the provided host if we have any + * @return {@link HostMetadata} about the provided host if we have any * metadata, {@code null} otherwise */ HostMetadata resolveMetadata(HttpHost host); } + /** + * Resolves metadata for all hosts to {@code null}, meaning "I have + * no metadata for this host". + */ + public static final HostMetadataResolver EMPTY_RESOLVER = new HostMetadataResolver() { + @Override + public HostMetadata resolveMetadata(HttpHost host) { + return null; + } + }; + /** + * Version of Elasticsearch that the host is running. + */ private final String version; + /** + * Roles that the Elasticsearch process on the host has. + */ private final Roles roles; public HostMetadata(String version, Roles roles) { @@ -59,7 +72,7 @@ public String version() { } /** - * Roles the node is implementing. + * Roles implemented by the Elasticsearch process. */ public Roles roles() { return roles; @@ -85,6 +98,9 @@ public int hashCode() { return Objects.hash(version, roles); } + /** + * Role information about and Elasticsearch process. + */ public static final class Roles { private final boolean master; private final boolean data; diff --git a/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java b/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java index 88174d65c9fa1..368956c98fb12 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java +++ b/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java @@ -21,6 +21,10 @@ import org.apache.http.HttpHost; +/** + * Selects hosts that can receive requests. Use with + * {@link RestClientActions#withHostSelector withHostSelector}. + */ public interface HostSelector { /** * Selector that matches any node. @@ -38,8 +42,9 @@ public String toString() { }; /** - * Selector that matches any node that doesn't have the master roles - * or doesn't have any information about roles. + * Selector that matches any node that has metadata and doesn't + * have the {@code master} role. These nodes cannot become a "master" + * node for the Elasticsearch cluster. */ HostSelector NOT_MASTER = new HostSelector() { @Override @@ -54,8 +59,12 @@ public String toString() { }; /** - * Return {@code true} if the provided host should be used for requests, {@code false} - * otherwise. + * Return {@code true} if the provided host should be used for requests, + * {@code false} otherwise. + * @param host the host being checked + * @param meta metadata about the host being checked or {@code null} if + * the {@linkplain RestClient} doesn't have any metadata about the + * host. */ boolean select(HttpHost host, HostMetadata meta); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java index 1a2985f6ef998..69d9a05e18f0f 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java @@ -24,7 +24,13 @@ import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; +/** + * Actions that can be taken on a {@link RestClient} or the stateless views + * returned by {@link RestClientActions#withHostSelector withHostSelector}. + */ public interface RestClientActions { /** * Sends a request to the Elasticsearch cluster that the client points to and waits for the corresponding response diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java index 616429dcec061..0a62cea59239b 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java @@ -27,13 +27,19 @@ import org.apache.http.HttpHost; /** - * Light weight view into a {@link RestClient} that doesn't have any state of its own. + * Stateless view into a {@link RestClient} some fixed parameters. */ class RestClientView extends AbstractRestClientActions { + /** + * {@linkplain RestClient} to which to delegate all requests. + */ private final RestClient delegate; + /** + * Selects which hosts are valid destinations requests. + */ private final HostSelector hostSelector; - protected RestClientView(RestClient delegate, HostSelector hostSelector) { + RestClientView(RestClient delegate, HostSelector hostSelector) { this.delegate = delegate; this.hostSelector = hostSelector; } From 17042221d8288ae0e1db6502ec4b3a5ca9d23f18 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 22 Mar 2018 15:18:45 -0400 Subject: [PATCH 11/27] Cleanups --- .../client/AbstractRestClientActions.java | 6 ++ .../org/elasticsearch/client/RestClient.java | 6 +- .../client/RestClientActions.java | 11 +++- .../client/RestClientBuilder.java | 6 +- .../elasticsearch/client/RestClientView.java | 2 +- .../RestClientMultipleHostsIntegTests.java | 59 ++++++++++++++++--- .../client/RestClientMultipleHostsTests.java | 13 ++++ 7 files changed, 90 insertions(+), 13 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java index ac3ad38d2b5e6..040b95e9a3e64 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java @@ -21,6 +21,7 @@ package org.elasticsearch.client; import java.io.IOException; +import java.net.ConnectException; import java.net.SocketTimeoutException; import java.util.Collections; import java.util.Map; @@ -168,6 +169,11 @@ Response get() throws IOException { if (exception instanceof ResponseException) { throw new ResponseException((ResponseException) exception); } + if (exception instanceof ConnectException) { + ConnectException e = new ConnectException(exception.getMessage()); + e.initCause(exception); + throw e; + } if (exception instanceof ConnectTimeoutException) { ConnectTimeoutException e = new ConnectTimeoutException(exception.getMessage()); e.initCause(exception); diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index b7aa952d2674f..ee5cdb4527a51 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -125,7 +125,8 @@ public static RestClientBuilder builder(HttpHost... hosts) { } /** - * Replaces the hosts that the client communicates with. + * Replaces the hosts that the client communicates with without + * changing any {@link HostMetadata}. * @see HttpHost */ public void setHosts(HttpHost... hosts) { @@ -162,6 +163,9 @@ public void setHosts(Iterable hosts, HostMetadataResolver metaResolver this.blacklist.clear(); } + /** + * Get the metadata resolver associated with this client. + */ public HostMetadataResolver getHostMetadataResolver() { return hostTuple.metaResolver; } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java index 69d9a05e18f0f..f927172d1b776 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java @@ -19,6 +19,7 @@ package org.elasticsearch.client; +import java.io.Closeable; import java.io.IOException; import java.util.Map; @@ -179,9 +180,13 @@ void performRequestAsync(String method, String endpoint, Map par ResponseListener responseListener, Header... headers); /** - * Create a client that only runs requests on selected hosts. This client - * has no state of its own and backs everything to the {@link RestClient} - * that created it. + * Create a "stateless view" of this client that only runs requests on + * selected hosts. The result of this method is "stateless" it backs all + * of its requests to the {@link RestClient} that created it so there is + * no need to manage this view with try-with-resources and it does not + * extend {@link Closeable}. Closing the {@linkplain RestClient} that + * created the view disposes of the underlying connection, making the + * view useless. */ RestClientActions withHostSelector(HostSelector hostSelector); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java index 5fd5afd6f89ea..2b5e450a6aae7 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java @@ -76,7 +76,11 @@ public final class RestClientBuilder { this.hosts = hosts; } - public RestClientBuilder setHostMetadata(HostMetadataResolver metaResolver) { + /** + * Set the {@link HostMetadataResolver} used when {@link HostSelector}s + * chose hosts. + */ + public RestClientBuilder setHostMetadataResolver(HostMetadataResolver metaResolver) { this.metaResolver = metaResolver; return this; } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java index 0a62cea59239b..b96e083121b27 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java @@ -27,7 +27,7 @@ import org.apache.http.HttpHost; /** - * Stateless view into a {@link RestClient} some fixed parameters. + * Stateless view into a {@link RestClient} with customized parameters. */ class RestClientView extends AbstractRestClientActions { /** diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java index da5a960c2e84c..1df78d6bc3273 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java @@ -25,6 +25,7 @@ import org.apache.http.HttpHost; import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement; import org.elasticsearch.mocksocket.MockHttpServer; +import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -41,8 +42,11 @@ import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes; import static org.elasticsearch.client.RestClientTestUtil.randomErrorNoRetryStatusCode; import static org.elasticsearch.client.RestClientTestUtil.randomOkStatusCode; +import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * Integration test to check interaction between {@link RestClient} and {@link org.apache.http.client.HttpClient}. @@ -53,12 +57,14 @@ public class RestClientMultipleHostsIntegTests extends RestClientTestCase { private static HttpServer[] httpServers; - private static RestClient restClient; + private static HttpHost[] httpHosts; + private static String pathPrefixWithoutLeadingSlash; private static String pathPrefix; + private RestClient restClient; + @BeforeClass public static void startHttpServer() throws Exception { - String pathPrefixWithoutLeadingSlash; if (randomBoolean()) { pathPrefixWithoutLeadingSlash = "testPathPrefix/" + randomAsciiOfLengthBetween(1, 5); pathPrefix = "/" + pathPrefixWithoutLeadingSlash; @@ -67,12 +73,17 @@ public static void startHttpServer() throws Exception { } int numHttpServers = randomIntBetween(2, 4); httpServers = new HttpServer[numHttpServers]; - HttpHost[] httpHosts = new HttpHost[numHttpServers]; + httpHosts = new HttpHost[numHttpServers]; for (int i = 0; i < numHttpServers; i++) { HttpServer httpServer = createHttpServer(); httpServers[i] = httpServer; httpHosts[i] = new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort()); } + httpServers[0].createContext(pathPrefix + "/firstOnly", new ResponseHandler(200)); + } + + @Before + public void buildRestClient() throws Exception { RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); if (pathPrefix.length() > 0) { restClientBuilder.setPathPrefix((randomBoolean() ? "/" : "") + pathPrefixWithoutLeadingSlash); @@ -107,18 +118,20 @@ public void handle(HttpExchange httpExchange) throws IOException { } } + @After + public void stopRestClient() throws IOException { + restClient.close(); + } + @AfterClass public static void stopHttpServers() throws IOException { - restClient.close(); - restClient = null; for (HttpServer httpServer : httpServers) { httpServer.stop(0); } httpServers = null; } - @Before - public void stopRandomHost() { + private void stopRandomHost() { //verify that shutting down some hosts doesn't matter as long as one working host is left behind if (httpServers.length > 1 && randomBoolean()) { List updatedHttpServers = new ArrayList<>(httpServers.length - 1); @@ -136,6 +149,7 @@ public void stopRandomHost() { } public void testSyncRequests() throws IOException { + stopRandomHost(); int numRequests = randomIntBetween(5, 20); for (int i = 0; i < numRequests; i++) { final String method = RestClientTestUtil.randomHttpMethod(getRandom()); @@ -154,6 +168,7 @@ public void testSyncRequests() throws IOException { } public void testAsyncRequests() throws Exception { + stopRandomHost(); int numRequests = randomIntBetween(5, 20); final CountDownLatch latch = new CountDownLatch(numRequests); final List responses = new CopyOnWriteArrayList<>(); @@ -187,6 +202,33 @@ public void onFailure(Exception exception) { } } + /** + * Test host selector against a real server and + * test what happens after calling + */ + public void testWithHostSelector() throws IOException { + /* + * note that this *doesn't* stopRandomHost(); because it might stop + * the first host which we use for testing the selector. + */ + HostSelector firstPositionOnly = new HostSelector() { + @Override + public boolean select(HttpHost host, HostMetadata meta) { + return httpHosts[0] == host; + } + }; + RestClientActions withHostSelector = restClient.withHostSelector(firstPositionOnly); + Response response = withHostSelector.performRequest("GET", "/firstOnly"); + assertEquals(httpHosts[0], response.getHost()); + restClient.close(); + try { + withHostSelector.performRequest("GET", "/firstOnly"); + fail("expected a failure"); + } catch (IllegalStateException e) { + assertThat(e.getMessage(), containsString("status: STOPPED")); + } + } + private static class TestResponse { private final String method; private final int statusCode; @@ -205,6 +247,9 @@ Response getResponse() { if (response instanceof ResponseException) { return ((ResponseException) response).getResponse(); } + if (response instanceof Exception) { + throw new AssertionError("unexpected response " + response.getClass(), (Exception) response); + } throw new AssertionError("unexpected response " + response.getClass()); } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java index 03561b0d5524c..393fb96e72b25 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java @@ -347,6 +347,19 @@ public HostMetadata resolveMetadata(HttpHost host) { assertEquals(httpHosts[0], response.getHost()); } + public void testWithHostSelector() throws IOException { + HostSelector firstPositionOnly = new HostSelector() { + @Override + public boolean select(HttpHost host, HostMetadata meta) { + return httpHosts[0] == host; + } + }; + RestClientActions withHostSelector = restClient.withHostSelector(firstPositionOnly); + Response response = withHostSelector.performRequest("GET", "/200"); + assertEquals(httpHosts[0], response.getHost()); + restClient.close(); + } + /** * Test that calling {@link RestClient#setHosts(Iterable)} preserves the * {@link HostMetadataResolver}. From 11708034a8d9bb9fc56d325b04feacd17357293e Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 22 Mar 2018 15:40:09 -0400 Subject: [PATCH 12/27] More cleanup --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 1 - .../test/rest/yaml/ClientYamlDocsTestClient.java | 4 ++-- .../test/rest/yaml/section/ClientYamlTestSection.java | 2 +- .../org/elasticsearch/test/rest/yaml/section/DoSection.java | 1 - 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 3ea3f1daf9573..df658ab9263bb 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -426,7 +426,6 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE final TimeValue socketTimeout = TimeValue.parseTimeValue(socketTimeoutString, CLIENT_SOCKET_TIMEOUT); builder.setRequestConfigCallback(conf -> conf.setSocketTimeout(Math.toIntExact(socketTimeout.getMillis()))); } - return builder.build(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java index c69ad4057c6d3..74a8479bfdfa4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java @@ -42,8 +42,8 @@ */ public final class ClientYamlDocsTestClient extends ClientYamlTestClient { - public ClientYamlDocsTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List hosts, - Version esVersion) throws IOException { + public ClientYamlDocsTestClient(ClientYamlSuiteRestSpec restSpec, RestClient restClient, List hosts, Version esVersion) + throws IOException { super(restSpec, restClient, hosts, esVersion); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index a15d498b106ab..09d9cb849812a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -98,7 +98,7 @@ public void addExecutableSection(ExecutableSection executableSection) { + "[skip] so runners that do not support the [host_selector] section can skip the test at line [" + doSection.getLocation().lineNumber + "]"); } - } + } this.executableSections.add(executableSection); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index b163cd79a948f..ac491e81a3755 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -197,7 +197,6 @@ public String toString() { public DoSection(XContentLocation location) { this.location = location; - } public String getCatch() { From c8d34587ae801d6e2cb0d714a2d8a3b4e6467102 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 22 Mar 2018 16:45:55 -0400 Subject: [PATCH 13/27] Cleanup --- .../java/org/elasticsearch/client/AbstractRestClientActions.java | 1 - .../main/java/org/elasticsearch/client/sniff/HostsSniffer.java | 1 - 2 files changed, 2 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java index 040b95e9a3e64..1579a918196c0 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java @@ -1,4 +1,3 @@ - /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java index e27ef1dd5f8b0..99865ea548c8f 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java @@ -20,7 +20,6 @@ package org.elasticsearch.client.sniff; import org.apache.http.HttpHost; -import org.elasticsearch.client.HostMetadata; import java.io.IOException; import java.util.Map; From 5f57bf4b3d56bd8e54ff7ef0fe2dcccedde193cb Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 23 Mar 2018 10:00:57 -0400 Subject: [PATCH 14/27] Fix javadoc --- .../main/java/org/elasticsearch/test/rest/ESRestTestCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index df658ab9263bb..108ecc1be3a90 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -39,6 +39,7 @@ import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer; +import org.elasticsearch.client.sniff.Sniffer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; From 36593c1ed5c15294adfb8b85afdb2f38112ba1c7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 26 Mar 2018 17:23:17 -0400 Subject: [PATCH 15/27] Switch from HostMetadata to Node Switches from storing metadata externally in the high level rest client to storing it *in* a `Node` abstraction that contains the `HttpHost`. --- .../elasticsearch/client/HostMetadata.java | 159 -------------- .../elasticsearch/client/HostSelector.java | 70 ------ .../java/org/elasticsearch/client/Node.java | 204 ++++++++++++++++++ .../elasticsearch/client/NodeSelector.java | 91 ++++++++ .../org/elasticsearch/client/RestClient.java | 200 +++++++++-------- .../client/RestClientActions.java | 4 +- .../client/RestClientBuilder.java | 33 +-- .../elasticsearch/client/RestClientView.java | 19 +- .../client/HostsTrackingFailureListener.java | 10 +- .../client/NodeSelectorTests.java | 50 +++++ .../org/elasticsearch/client/NodeTests.java | 61 ++++++ .../client/RestClientBuilderTests.java | 31 ++- .../RestClientMultipleHostsIntegTests.java | 17 +- .../client/RestClientMultipleHostsTests.java | 121 ++++------- .../client/RestClientSingleHostTests.java | 62 +++--- .../elasticsearch/client/RestClientTests.java | 99 ++++----- .../RestClientDocumentation.java | 8 +- .../sniff/ElasticsearchHostsSniffer.java | 26 +-- .../client/sniff/HostsSniffer.java | 4 +- .../elasticsearch/client/sniff/Sniffer.java | 25 +-- .../client/sniff/SnifferResult.java | 77 ------- .../sniff/ElasticsearchHostsSnifferTests.java | 54 ++--- .../client/sniff/MockHostsSniffer.java | 13 +- .../documentation/SnifferDocumentation.java | 5 +- .../rest-api-spec/test/index/10_with_id.yml | 6 +- .../test/rest/ESRestTestCase.java | 72 +++++-- .../rest/yaml/ClientYamlDocsTestClient.java | 6 +- .../test/rest/yaml/ClientYamlTestClient.java | 6 +- .../yaml/ClientYamlTestExecutionContext.java | 22 +- .../test/rest/yaml/Features.java | 2 +- .../rest/yaml/section/ApiCallSection.java | 12 +- .../yaml/section/ClientYamlTestSection.java | 10 +- .../test/rest/yaml/section/DoSection.java | 42 ++-- .../test/rest/ESRestTestCaseTests.java | 86 ++++++++ .../ClientYamlTestExecutionContextTests.java | 12 +- .../section/ClientYamlTestSectionTests.java | 16 +- .../rest/yaml/section/DoSectionTests.java | 32 +-- 37 files changed, 993 insertions(+), 774 deletions(-) delete mode 100644 client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java delete mode 100644 client/rest/src/main/java/org/elasticsearch/client/HostSelector.java create mode 100644 client/rest/src/main/java/org/elasticsearch/client/Node.java create mode 100644 client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java create mode 100644 client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java create mode 100644 client/rest/src/test/java/org/elasticsearch/client/NodeTests.java delete mode 100644 client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java create mode 100644 test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java diff --git a/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java b/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java deleted file mode 100644 index 19e87b674d194..0000000000000 --- a/client/rest/src/main/java/org/elasticsearch/client/HostMetadata.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client; - -import java.util.Objects; - -import org.apache.http.HttpHost; - -/** - * Metadata about an {@link HttpHost} running Elasticsearch. - */ -public class HostMetadata { - /** - * Look up metadata about the provided host. Implementers should not - * make network calls, instead, they should look up previously fetched - * data. See Elasticsearch's Sniffer for an example implementation. - */ - public interface HostMetadataResolver { - /** - * @return {@link HostMetadata} about the provided host if we have any - * metadata, {@code null} otherwise - */ - HostMetadata resolveMetadata(HttpHost host); - } - /** - * Resolves metadata for all hosts to {@code null}, meaning "I have - * no metadata for this host". - */ - public static final HostMetadataResolver EMPTY_RESOLVER = new HostMetadataResolver() { - @Override - public HostMetadata resolveMetadata(HttpHost host) { - return null; - } - }; - - /** - * Version of Elasticsearch that the host is running. - */ - private final String version; - /** - * Roles that the Elasticsearch process on the host has. - */ - private final Roles roles; - - public HostMetadata(String version, Roles roles) { - this.version = Objects.requireNonNull(version, "version is required"); - this.roles = Objects.requireNonNull(roles, "roles is required"); - } - - /** - * Version of the node. - */ - public String version() { - return version; - } - - /** - * Roles implemented by the Elasticsearch process. - */ - public Roles roles() { - return roles; - } - - @Override - public String toString() { - return "[version=" + version + ", roles=" + roles + "]"; - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - HostMetadata other = (HostMetadata) obj; - return version.equals(other.version) - && roles.equals(other.roles); - } - - @Override - public int hashCode() { - return Objects.hash(version, roles); - } - - /** - * Role information about and Elasticsearch process. - */ - public static final class Roles { - private final boolean master; - private final boolean data; - private final boolean ingest; - - public Roles(boolean master, boolean data, boolean ingest) { - this.master = master; - this.data = data; - this.ingest = ingest; - } - - /** - * The node could be elected master. - */ - public boolean master() { - return master; - } - /** - * The node stores data. - */ - public boolean data() { - return data; - } - /** - * The node runs ingest pipelines. - */ - public boolean ingest() { - return ingest; - } - - @Override - public String toString() { - StringBuilder result = new StringBuilder(3); - if (master) result.append('m'); - if (data) result.append('d'); - if (ingest) result.append('i'); - return result.toString(); - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - Roles other = (Roles) obj; - return master == other.master - && data == other.data - && ingest == other.ingest; - } - - @Override - public int hashCode() { - return Objects.hash(master, data, ingest); - } - } -} diff --git a/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java b/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java deleted file mode 100644 index 368956c98fb12..0000000000000 --- a/client/rest/src/main/java/org/elasticsearch/client/HostSelector.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client; - -import org.apache.http.HttpHost; - -/** - * Selects hosts that can receive requests. Use with - * {@link RestClientActions#withHostSelector withHostSelector}. - */ -public interface HostSelector { - /** - * Selector that matches any node. - */ - HostSelector ANY = new HostSelector() { - @Override - public boolean select(HttpHost host, HostMetadata meta) { - return true; - } - - @Override - public String toString() { - return "ANY"; - } - }; - - /** - * Selector that matches any node that has metadata and doesn't - * have the {@code master} role. These nodes cannot become a "master" - * node for the Elasticsearch cluster. - */ - HostSelector NOT_MASTER = new HostSelector() { - @Override - public boolean select(HttpHost host, HostMetadata meta) { - return meta != null && false == meta.roles().master(); - } - - @Override - public String toString() { - return "NOT_MASTER"; - } - }; - - /** - * Return {@code true} if the provided host should be used for requests, - * {@code false} otherwise. - * @param host the host being checked - * @param meta metadata about the host being checked or {@code null} if - * the {@linkplain RestClient} doesn't have any metadata about the - * host. - */ - boolean select(HttpHost host, HostMetadata meta); -} diff --git a/client/rest/src/main/java/org/elasticsearch/client/Node.java b/client/rest/src/main/java/org/elasticsearch/client/Node.java new file mode 100644 index 0000000000000..833bc2b05f12e --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/Node.java @@ -0,0 +1,204 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.util.List; +import java.util.Objects; + +import org.apache.http.HttpHost; + +/** + * Metadata about an {@link HttpHost} running Elasticsearch. + */ +public class Node { + /** + * Address that this host claims is its primary contact point. + */ + private final HttpHost host; + /** + * Addresses that this host is bound to. + */ + private final List boundHosts; + /** + * Version of Elasticsearch that the node is running or {@code null} + * if we don't know the version. + */ + private final String version; + /** + * Roles that the Elasticsearch process on the host has or {@code null} + * if we don't know what roles the node has. + */ + private final Roles roles; + + /** + * Create a {@linkplain Node} with metadata. + */ + public Node(HttpHost host, List boundHosts, String version, Roles roles) { + if (host == null) { + throw new IllegalArgumentException("host cannot be null"); + } + this.host = host; + this.boundHosts = boundHosts; + this.version = version; + this.roles = roles; + } + + /** + * Create a {@linkplain Node} without any metadata. + */ + public Node(HttpHost host) { + this(host, null, null, null); + } + + /** + * Contact information for the host. + */ + public HttpHost getHost() { + return host; + } + + /** + * Addresses that this host is bound to. + */ + public List getBoundHosts() { + return boundHosts; + } + + /** + * Version of Elasticsearch that the node is running or {@code null} + * if we don't know the version. + */ + public String getVersion() { + return version; + } + + /** + * Roles that the Elasticsearch process on the host has or {@code null} + * if we don't know what roles the node has. + */ + public Roles getRoles() { + return roles; + } + + /** + * Make a copy of this {@link Node} but replacing its {@link #getHost()} + * with the provided {@link HttpHost}. The provided host must be part of + * of {@link #getBoundHosts() bound hosts}. + */ + public Node withBoundHostAsHost(HttpHost boundHost) { + if (false == boundHosts.contains(boundHost)) { + throw new IllegalArgumentException(boundHost + " must be a bound host but wasn't in " + + boundHosts); + } + return new Node(boundHost, boundHosts, version, roles); + } + + @Override + public String toString() { + StringBuilder b = new StringBuilder(); + b.append("[host=").append(host); + if (boundHosts != null) { + b.append(", bound=").append(boundHosts); + } + if (version != null) { + b.append(", version=").append(version); + } + if (roles != null) { + b.append(", roles=").append(roles); + } + return b.append(']').toString(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + Node other = (Node) obj; + return host.equals(other.host) + && Objects.equals(version, other.version) + && Objects.equals(roles, other.roles) + && Objects.equals(boundHosts, other.boundHosts); + } + + @Override + public int hashCode() { + return Objects.hash(host, version, roles, boundHosts); + } + + /** + * Role information about an Elasticsearch process. + */ + public static final class Roles { + private final boolean master; + private final boolean data; + private final boolean ingest; + + public Roles(boolean master, boolean data, boolean ingest) { + this.master = master; + this.data = data; + this.ingest = ingest; + } + + /** + * The node could be elected master. + */ + public boolean hasMaster() { + return master; + } + /** + * The node stores data. + */ + public boolean hasData() { + return data; + } + /** + * The node runs ingest pipelines. + */ + public boolean hasIngest() { + return ingest; + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(3); + if (master) result.append('m'); + if (data) result.append('d'); + if (ingest) result.append('i'); + return result.toString(); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + Roles other = (Roles) obj; + return master == other.master + && data == other.data + && ingest == other.ingest; + } + + @Override + public int hashCode() { + return Objects.hash(master, data, ingest); + } + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java new file mode 100644 index 0000000000000..889a6b2920972 --- /dev/null +++ b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import java.util.Objects; + +/** + * Selects nodes that can receive requests. Use with + * {@link RestClientActions#withNodeSelector withNodeSelector}. + */ +public interface NodeSelector { + /** + * Return {@code true} if the provided node should be used for requests, + * {@code false} otherwise. + */ + boolean select(Node node); + + /** + * Selector that matches any node. + */ + NodeSelector ANY = new NodeSelector() { + @Override + public boolean select(Node node) { + return true; + } + + @Override + public String toString() { + return "ANY"; + } + }; + + /** + * Selector that matches any node that has metadata and doesn't + * have the {@code master} role OR it has the data {@code data} + * role. + */ + NodeSelector NOT_MASTER_ONLY = new NodeSelector() { + @Override + public boolean select(Node node) { + return node.getRoles() != null + && (false == node.getRoles().hasMaster() || node.getRoles().hasData()); + } + + @Override + public String toString() { + return "NOT_MASTER_ONLY"; + } + }; + + /** + * Selector that returns {@code true} of both of its provided + * selectors return {@code true}, otherwise {@code false}. + */ + class And implements NodeSelector { + private final NodeSelector lhs; + private final NodeSelector rhs; + + public And(NodeSelector lhs, NodeSelector rhs) { + this.lhs = Objects.requireNonNull(lhs, "lhs is required"); + this.rhs = Objects.requireNonNull(rhs, "rhs is required"); + } + + @Override + public boolean select(Node node) { + return lhs.select(node) && rhs.select(node); + } + + @Override + public String toString() { + return lhs + " AND " + rhs; + } + } +} diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index ee5cdb4527a51..4851580143e72 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -44,7 +44,6 @@ import org.apache.http.nio.client.methods.HttpAsyncMethods; import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; -import org.elasticsearch.client.HostMetadata.HostMetadataResolver; import java.io.Closeable; import java.io.IOException; @@ -90,10 +89,10 @@ public class RestClient extends AbstractRestClientActions implements Closeable { private static final Log logger = LogFactory.getLog(RestClient.class); /** - * The maximum number of attempts that {@link #nextHost(HostSelector)} makes + * The maximum number of attempts that {@link #nextNode(NodeSelector)} makes * before giving up and failing the request. */ - private static final int MAX_NEXT_HOSTS_ATTEMPTS = 10; + private static final int MAX_NEXT_NODES_ATTEMPTS = 10; private final CloseableHttpAsyncClient client; // We don't rely on default headers supported by HttpAsyncClient as those cannot be replaced. @@ -101,73 +100,86 @@ public class RestClient extends AbstractRestClientActions implements Closeable { final List

defaultHeaders; private final long maxRetryTimeoutMillis; private final String pathPrefix; - private final AtomicInteger lastHostIndex = new AtomicInteger(0); + private final AtomicInteger lastNodeIndex = new AtomicInteger(0); private final ConcurrentMap blacklist = new ConcurrentHashMap<>(); private final FailureListener failureListener; - private volatile HostTuple> hostTuple; + private volatile NodeTuple> nodeTuple; RestClient(CloseableHttpAsyncClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders, - HttpHost[] hosts, HostMetadataResolver metaResolver, String pathPrefix, FailureListener failureListener) { + Node[] nodes, String pathPrefix, FailureListener failureListener) { this.client = client; this.maxRetryTimeoutMillis = maxRetryTimeoutMillis; this.defaultHeaders = Collections.unmodifiableList(Arrays.asList(defaultHeaders)); this.failureListener = failureListener; this.pathPrefix = pathPrefix; - setHosts(Arrays.asList(hosts), metaResolver); + setNodes(nodes); } /** * Returns a new {@link RestClientBuilder} to help with {@link RestClient} creation. - * Creates a new builder instance and sets the hosts that the client will send requests to. + * Creates a new builder instance and sets the nodes that the client will send requests to. + * @see Node#Node(HttpHost) */ public static RestClientBuilder builder(HttpHost... hosts) { - return new RestClientBuilder(hosts); + return builder(hostsToNodes(hosts)); } /** - * Replaces the hosts that the client communicates with without - * changing any {@link HostMetadata}. - * @see HttpHost + * Returns a new {@link RestClientBuilder} to help with {@link RestClient} creation. + * Creates a new builder instance and sets the nodes that the client will send requests to. + */ + public static RestClientBuilder builder(Node... nodes) { + return new RestClientBuilder(nodes); + } + + /** + * Replaces the nodes that the client communicates without providing any + * metadata about any of the nodes. + * @see Node#Node(HttpHost) */ public void setHosts(HttpHost... hosts) { - if (hosts == null) { - throw new IllegalArgumentException("hosts must not be null"); + setNodes(hostsToNodes(hosts)); + } + + private static Node[] hostsToNodes(HttpHost[] hosts) { + if (hosts == null || hosts.length == 0) { + throw new IllegalArgumentException("hosts must not be null or empty"); } - setHosts(Arrays.asList(hosts), hostTuple.metaResolver); + Node[] nodes = new Node[hosts.length]; + for (int i = 0; i < hosts.length; i++) { + nodes[i] = new Node(hosts[i]); + } + return nodes; } /** - * Replaces the hosts that the client communicates with and the - * {@link HostMetadata} used by any {@link HostSelector}s. - * @see HttpHost + * Replaces the nodes that the client communicates with. Prefer this to + * {@link #setHosts(HttpHost...)} if you have metadata about the hosts + * like their Elasticsearch version of which roles they implement. */ - public void setHosts(Iterable hosts, HostMetadataResolver metaResolver) { - if (hosts == null) { - throw new IllegalArgumentException("hosts must not be null"); - } - if (metaResolver == null) { - throw new IllegalArgumentException("metaResolver must not be null"); + public void setNodes(Node... nodes) { + if (nodes == null || nodes.length == 0) { + throw new IllegalArgumentException("nodes must not be null or empty"); } - Set newHosts = new HashSet<>(); + Set newNodes = new HashSet<>(); AuthCache authCache = new BasicAuthCache(); - for (HttpHost host : hosts) { - Objects.requireNonNull(host, "host cannot be null"); - newHosts.add(host); - authCache.put(host, new BasicScheme()); - } - if (newHosts.isEmpty()) { - throw new IllegalArgumentException("hosts must not be empty"); + for (Node node : nodes) { + if (node == null) { + throw new IllegalArgumentException("node cannot be null"); + } + newNodes.add(node); + authCache.put(node.getHost(), new BasicScheme()); } - this.hostTuple = new HostTuple<>(Collections.unmodifiableSet(newHosts), authCache, metaResolver); + this.nodeTuple = new NodeTuple<>(Collections.unmodifiableSet(newNodes), authCache); this.blacklist.clear(); } /** - * Get the metadata resolver associated with this client. + * Copy of the list of nodes that the client knows about. */ - public HostMetadataResolver getHostMetadataResolver() { - return hostTuple.metaResolver; + public Node[] getNodes() { // TODO is it ok to expose this? It feels excessive but we do use it in tests. + return nodeTuple.nodes.toArray(new Node[0]); } @Override @@ -176,8 +188,8 @@ final SyncResponseListener syncResponseListener() { } @Override - public RestClientActions withHostSelector(HostSelector hostSelector) { - return new RestClientView(this, hostSelector); + public RestClientActions withNodeSelector(NodeSelector nodeSelector) { + return new RestClientView(this, nodeSelector); } // TODO this exists entirely to so we don't have to change much in the high level rest client tests. We'll remove in a followup. @@ -198,15 +210,14 @@ public void performRequestAsync(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, ResponseListener responseListener, Header[] headers) throws IOException { - // Requests made directly to the client use the noop HostSelector. - HostSelector hostSelector = HostSelector.ANY; + // Requests made directly to the client use the noop NodeSelector. performRequestAsyncNoCatch(method, endpoint, params, entity, httpAsyncResponseConsumerFactory, - responseListener, hostSelector, headers); + responseListener, NodeSelector.ANY, headers); } void performRequestAsyncNoCatch(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, - ResponseListener responseListener, HostSelector hostSelector, Header[] headers) throws IOException { + ResponseListener responseListener, NodeSelector nodeSelector, Header[] headers) throws IOException { Objects.requireNonNull(params, "params must not be null"); Map requestParams = new HashMap<>(params); //ignore is a special parameter supported by the clients, shouldn't be sent to es @@ -239,15 +250,15 @@ void performRequestAsyncNoCatch(String method, String endpoint, Map> hostTuple, final HttpRequestBase request, + private void performRequestAsync(final long startTime, final NodeTuple> hostTuple, final HttpRequestBase request, final Set ignoreErrorCodes, final HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, final FailureTrackingResponseListener listener) { - final HttpHost host = hostTuple.hosts.next(); + final HttpHost host = hostTuple.nodes.next(); //we stream the request body if the entity allows for it final HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request); final HttpAsyncResponseConsumer asyncResponseConsumer = @@ -293,7 +304,7 @@ public void failed(Exception failure) { } private void retryIfPossible(Exception exception) { - if (hostTuple.hosts.hasNext()) { + if (hostTuple.nodes.hasNext()) { //in case we are retrying, check whether maxRetryTimeout has been reached long timeElapsedMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); long timeout = maxRetryTimeoutMillis - timeElapsedMillis; @@ -335,20 +346,20 @@ private void setHeaders(HttpRequest httpRequest, Header[] requestHeaders) { } /** - * Returns a non-empty {@link Iterator} of hosts to be used for a request - * that match the {@link HostSelector}. + * Returns a non-empty {@link Iterator} of nodes to be used for a request + * that match the {@link NodeSelector}. *

- * If there are no living hosts that match the {@link HostSelector} - * this will return the dead host that matches the {@link HostSelector} + * If there are no living nodes that match the {@link NodeSelector} + * this will return the dead node that matches the {@link NodeSelector} * that is closest to being revived. *

- * If no living and no dead hosts match the selector we retry a few - * times to handle concurrent modifications of the list of dead hosts. + * If no living and no dead nodes match the selector we retry a few + * times to handle concurrent modifications of the list of dead nodes. * We never block the thread or {@link Thread#sleep} or anything like * that. If the retries fail this throws a {@link IOException}. - * @throws IOException if no hosts are available + * @throws IOException if no nodes are available */ - private HostTuple> nextHost(HostSelector hostSelector) throws IOException { + private NodeTuple> nextNode(NodeSelector nodeSelector) throws IOException { int attempts = 0; NextHostsResult result; /* @@ -357,40 +368,40 @@ private HostTuple> nextHost(HostSelector hostSelector) throws * blacklist. */ do { - final HostTuple> hostTuple = this.hostTuple; - result = nextHostsOneTime(hostTuple, blacklist, lastHostIndex, System.nanoTime(), hostSelector); + final NodeTuple> nodeTuple = this.nodeTuple; + result = nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, System.nanoTime(), nodeSelector); if (result.hosts == null) { if (logger.isDebugEnabled()) { - logger.debug("No hosts avialable. Will retry. Failure is " + result.describeFailure()); + logger.debug("No nodes avialable. Will retry. Failure is " + result.describeFailure()); } } else { // Success! - return new HostTuple<>(result.hosts.iterator(), hostTuple.authCache, hostTuple.metaResolver); + return new NodeTuple<>(result.hosts.iterator(), nodeTuple.authCache); } attempts++; - } while (attempts < MAX_NEXT_HOSTS_ATTEMPTS); - throw new IOException("No hosts available for request. Last failure was " + result.describeFailure()); + } while (attempts < MAX_NEXT_NODES_ATTEMPTS); + throw new IOException("No nodes available for request. Last failure was " + result.describeFailure()); } static class NextHostsResult { /** - * Number of hosts filtered from the list because they are + * Number of nodes filtered from the list because they are * dead. */ int blacklisted = 0; /** - * Number of hosts filtered from the list because the. - * {@link HostSelector} didn't approve of them. + * Number of nodes filtered from the list because the. + * {@link NodeSelector} didn't approve of them. */ int selectorRejected = 0; /** - * Number of hosts that could not be revived because the - * {@link HostSelector} didn't approve of them. + * Number of nodes that could not be revived because the + * {@link NodeSelector} didn't approve of them. */ int selectorBlockedRevival = 0; /** - * {@code null} if we failed to find any hosts, a list of - * hosts to use if we found any. + * {@code null} if we failed to find any nodes, a list of + * nodes to use if we found any. */ Collection hosts = null; @@ -401,40 +412,48 @@ public String describeFailure() { + ", selectorBlockedRevival=" + selectorBlockedRevival + "]]"; } } - static NextHostsResult nextHostsOneTime(HostTuple> hostTuple, - Map blacklist, AtomicInteger lastHostIndex, - long now, HostSelector hostSelector) { + static NextHostsResult nextHostsOneTime(NodeTuple> nodeTuple, + Map blacklist, AtomicInteger lastNodesIndex, + long now, NodeSelector nodeSelector) { NextHostsResult result = new NextHostsResult(); - Set filteredHosts = new HashSet<>(hostTuple.hosts); + // TODO there has to be a better way! + Map hostToNode = new HashMap<>(nodeTuple.nodes.size()); + for (Node node : nodeTuple.nodes) { + hostToNode.put(node.getHost(), node); + } + Set filteredNodes = new HashSet<>(nodeTuple.nodes); for (Map.Entry entry : blacklist.entrySet()) { if (now - entry.getValue().getDeadUntilNanos() < 0) { - filteredHosts.remove(entry.getKey()); + filteredNodes.remove(hostToNode.get(entry.getKey())); result.blacklisted++; } } - for (Iterator hostItr = filteredHosts.iterator(); hostItr.hasNext();) { - final HttpHost host = hostItr.next(); - if (false == hostSelector.select(host, hostTuple.metaResolver.resolveMetadata(host))) { - hostItr.remove(); + for (Iterator nodeItr = filteredNodes.iterator(); nodeItr.hasNext();) { + final Node node = nodeItr.next(); + if (false == nodeSelector.select(node)) { + nodeItr.remove(); result.selectorRejected++; } } - if (false == filteredHosts.isEmpty()) { + if (false == filteredNodes.isEmpty()) { /* - * Normal case: we have at least one non-dead host that the hostSelector + * Normal case: we have at least one non-dead node that the nodeSelector * is fine with. Rotate the list so repeated requests with the same blacklist - * and the same selector round robin. If you use a different HostSelector - * or a host goes dark then the round robin won't be perfect but that should + * and the same selector round robin. If you use a different NodeSelector + * or a node goes dark then the round robin won't be perfect but that should * be fine. */ - List rotatedHosts = new ArrayList<>(filteredHosts); - int i = lastHostIndex.getAndIncrement(); + List rotatedHosts = new ArrayList<>(filteredNodes.size()); + for (Node node : filteredNodes) { + rotatedHosts.add(node.getHost()); + } + int i = lastNodesIndex.getAndIncrement(); Collections.rotate(rotatedHosts, i); result.hosts = rotatedHosts; return result; } /* - * Last resort: If there are no good hosts to use, return a single dead one, + * Last resort: If there are no good nodes to use, return a single dead one, * the one that's closest to being retried *and* matches the selector. */ List> sortedHosts = new ArrayList<>(blacklist.entrySet()); @@ -451,7 +470,8 @@ public int compare(Map.Entry o1, Map.Entry> nodeItr = sortedHosts.iterator(); while (nodeItr.hasNext()) { final HttpHost deadHost = nodeItr.next().getKey(); - if (hostSelector.select(deadHost, hostTuple.metaResolver.resolveMetadata(deadHost))) { + Node node = hostToNode.get(deadHost); + if (node != null && nodeSelector.select(node)) { if (logger.isTraceEnabled()) { logger.trace("resurrecting host [" + deadHost + "]"); } @@ -630,18 +650,16 @@ public void onFailure(HttpHost host) { } /** - * {@code HostTuple} enables the {@linkplain HttpHost}s and {@linkplain AuthCache} to be set together in a thread + * {@code HostTuple} enables the {@linkplain Node}s and {@linkplain AuthCache} to be set together in a thread * safe, volatile way. */ - static class HostTuple { - final T hosts; + static class NodeTuple { + final T nodes; final AuthCache authCache; - final HostMetadataResolver metaResolver; - HostTuple(final T hosts, final AuthCache authCache, final HostMetadataResolver metaResolver) { - this.hosts = hosts; + NodeTuple(final T nodes, final AuthCache authCache) { + this.nodes = nodes; this.authCache = authCache; - this.metaResolver = metaResolver; } } } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java index f927172d1b776..550f833eca55f 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java @@ -30,7 +30,7 @@ /** * Actions that can be taken on a {@link RestClient} or the stateless views - * returned by {@link RestClientActions#withHostSelector withHostSelector}. + * returned by {@link RestClientActions#withNodeSelector withNodeSelector}. */ public interface RestClientActions { /** @@ -188,5 +188,5 @@ void performRequestAsync(String method, String endpoint, Map par * created the view disposes of the underlying connection, making the * view useless. */ - RestClientActions withHostSelector(HostSelector hostSelector); + RestClientActions withNodeSelector(NodeSelector nodeSelector); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java index 2b5e450a6aae7..0f0acc4e420be 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientBuilder.java @@ -20,14 +20,12 @@ package org.elasticsearch.client; import org.apache.http.Header; -import org.apache.http.HttpHost; import org.apache.http.client.config.RequestConfig; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; import org.apache.http.nio.conn.SchemeIOSessionStrategy; -import org.elasticsearch.client.HostMetadata.HostMetadataResolver; import javax.net.ssl.SSLContext; import java.security.AccessController; @@ -50,8 +48,7 @@ public final class RestClientBuilder { private static final Header[] EMPTY_HEADERS = new Header[0]; - private final HttpHost[] hosts; - private HostMetadataResolver metaResolver = HostMetadata.EMPTY_RESOLVER; + private final Node[] nodes; private int maxRetryTimeout = DEFAULT_MAX_RETRY_TIMEOUT_MILLIS; private Header[] defaultHeaders = EMPTY_HEADERS; private RestClient.FailureListener failureListener; @@ -65,24 +62,16 @@ public final class RestClientBuilder { * @throws NullPointerException if {@code hosts} or any host is {@code null}. * @throws IllegalArgumentException if {@code hosts} is empty. */ - RestClientBuilder(HttpHost... hosts) { - Objects.requireNonNull(hosts, "hosts must not be null"); - if (hosts.length == 0) { - throw new IllegalArgumentException("no hosts provided"); + RestClientBuilder(Node[] nodes) { + if (nodes == null || nodes.length == 0) { + throw new IllegalArgumentException("nodes must not be null or empty"); } - for (HttpHost host : hosts) { - Objects.requireNonNull(host, "host cannot be null"); + for (Node node : nodes) { + if (node == null) { + throw new IllegalArgumentException("node cannot be null"); + } } - this.hosts = hosts; - } - - /** - * Set the {@link HostMetadataResolver} used when {@link HostSelector}s - * chose hosts. - */ - public RestClientBuilder setHostMetadataResolver(HostMetadataResolver metaResolver) { - this.metaResolver = metaResolver; - return this; + this.nodes = nodes; } /** @@ -198,8 +187,8 @@ public CloseableHttpAsyncClient run() { return createHttpClient(); } }); - RestClient restClient = new RestClient(httpClient, maxRetryTimeout, defaultHeaders, hosts, - metaResolver, pathPrefix, failureListener); + RestClient restClient = new RestClient(httpClient, maxRetryTimeout, defaultHeaders, nodes, + pathPrefix, failureListener); httpClient.start(); return restClient; } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java index b96e083121b27..0075241a9e690 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java @@ -37,11 +37,11 @@ class RestClientView extends AbstractRestClientActions { /** * Selects which hosts are valid destinations requests. */ - private final HostSelector hostSelector; + private final NodeSelector nodeSelector; - RestClientView(RestClient delegate, HostSelector hostSelector) { + RestClientView(RestClient delegate, NodeSelector nodeSelector) { this.delegate = delegate; - this.hostSelector = hostSelector; + this.nodeSelector = nodeSelector; } @Override @@ -50,15 +50,8 @@ final SyncResponseListener syncResponseListener() { } @Override - public final RestClientView withHostSelector(final HostSelector hostSelector) { - final HostSelector inner = this.hostSelector; - HostSelector combo = new HostSelector() { - @Override - public boolean select(HttpHost host, HostMetadata meta) { - return inner.select(host, meta) && hostSelector.select(host, meta); - } - }; - return new RestClientView(delegate, combo); + public final RestClientView withNodeSelector(final NodeSelector newNodeSelector) { + return new RestClientView(delegate, new NodeSelector.And(nodeSelector, newNodeSelector)); } @Override @@ -66,6 +59,6 @@ final void performRequestAsyncNoCatch(String method, String endpoint, MapemptyList(), + randomAsciiAlphanumOfLength(5), new Roles(master, data, ingest)); + } +} diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java new file mode 100644 index 0000000000000..fecc0d379aa2c --- /dev/null +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java @@ -0,0 +1,61 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.Node.Roles; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class NodeTests extends RestClientTestCase { + public void testWithBoundHostAsHost() { + HttpHost h1 = new HttpHost("1"); + HttpHost h2 = new HttpHost("2"); + HttpHost h3 = new HttpHost("3"); + + Node n = new Node(h1, Arrays.asList(h1, h2, h3), randomAsciiAlphanumOfLength(5), + new Roles(randomBoolean(), randomBoolean(), randomBoolean())); + assertEquals(h2, n.withBoundHostAsHost(h2).getHost()); + assertEquals(n.getBoundHosts(), n.withBoundHostAsHost(h2).getBoundHosts()); + + try { + n.withBoundHostAsHost(new HttpHost("4")); + fail("expected failure"); + } catch (IllegalArgumentException e) { + assertEquals("http://4 must be a bound host but wasn't in [http://1, http://2, http://3]", + e.getMessage()); + } + } + + public void testToString() { + assertEquals("[host=http://1]", new Node(new HttpHost("1")).toString()); + assertEquals("[host=http://1, roles=mdi]", new Node(new HttpHost("1"), + null, null, new Roles(true, true, true)).toString()); + assertEquals("[host=http://1, version=ver]", new Node(new HttpHost("1"), + null, "ver", null).toString()); + assertEquals("[host=http://1, bound=[http://1, http://2]]", new Node(new HttpHost("1"), + Arrays.asList(new HttpHost("1"), new HttpHost("2")), null, null).toString()); + } + + // TODO tests for equals and hashcode probably +} diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java index c9243d3aaf6ce..b11d240f27bd4 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderTests.java @@ -39,24 +39,45 @@ public void testBuild() throws IOException { try { RestClient.builder((HttpHost[])null); fail("should have failed"); - } catch(NullPointerException e) { - assertEquals("hosts must not be null", e.getMessage()); + } catch(IllegalArgumentException e) { + assertEquals("hosts must not be null or empty", e.getMessage()); } try { - RestClient.builder(); + RestClient.builder(new HttpHost[] {}); fail("should have failed"); } catch(IllegalArgumentException e) { - assertEquals("no hosts provided", e.getMessage()); + assertEquals("hosts must not be null or empty", e.getMessage()); } try { RestClient.builder(new HttpHost("localhost", 9200), null); fail("should have failed"); - } catch(NullPointerException e) { + } catch(IllegalArgumentException e) { assertEquals("host cannot be null", e.getMessage()); } + try { + RestClient.builder((Node[])null); + fail("should have failed"); + } catch(IllegalArgumentException e) { + assertEquals("nodes must not be null or empty", e.getMessage()); + } + + try { + RestClient.builder(new Node[] {}); + fail("should have failed"); + } catch(IllegalArgumentException e) { + assertEquals("nodes must not be null or empty", e.getMessage()); + } + + try { + RestClient.builder(new Node(new HttpHost("localhost", 9200)), null); + fail("should have failed"); + } catch(IllegalArgumentException e) { + assertEquals("node cannot be null", e.getMessage()); + } + try (RestClient restClient = RestClient.builder(new HttpHost("localhost", 9200)).build()) { assertNotNull(restClient); } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java index 1df78d6bc3273..3ef8b3274acd4 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java @@ -66,7 +66,8 @@ public class RestClientMultipleHostsIntegTests extends RestClientTestCase { @BeforeClass public static void startHttpServer() throws Exception { if (randomBoolean()) { - pathPrefixWithoutLeadingSlash = "testPathPrefix/" + randomAsciiOfLengthBetween(1, 5); + pathPrefixWithoutLeadingSlash = "testPathPrefix/" + + randomAsciiLettersOfLengthBetween(1, 5); pathPrefix = "/" + pathPrefixWithoutLeadingSlash; } else { pathPrefix = pathPrefixWithoutLeadingSlash = ""; @@ -206,23 +207,23 @@ public void onFailure(Exception exception) { * Test host selector against a real server and * test what happens after calling */ - public void testWithHostSelector() throws IOException { + public void testWithNodeSelector() throws IOException { /* * note that this *doesn't* stopRandomHost(); because it might stop * the first host which we use for testing the selector. */ - HostSelector firstPositionOnly = new HostSelector() { + NodeSelector firstPositionOnly = new NodeSelector() { @Override - public boolean select(HttpHost host, HostMetadata meta) { - return httpHosts[0] == host; + public boolean select(Node node) { + return httpHosts[0] == node.getHost(); } }; - RestClientActions withHostSelector = restClient.withHostSelector(firstPositionOnly); - Response response = withHostSelector.performRequest("GET", "/firstOnly"); + RestClientActions withNodeSelector = restClient.withNodeSelector(firstPositionOnly); + Response response = withNodeSelector.performRequest("GET", "/firstOnly"); assertEquals(httpHosts[0], response.getHost()); restClient.close(); try { - withHostSelector.performRequest("GET", "/firstOnly"); + withNodeSelector.performRequest("GET", "/firstOnly"); fail("expected a failure"); } catch (IllegalStateException e) { assertThat(e.getMessage(), containsString("status: STOPPED")); diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java index 393fb96e72b25..97a171a3a4e00 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java @@ -35,7 +35,7 @@ import org.apache.http.message.BasicStatusLine; import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; -import org.elasticsearch.client.HostMetadata.HostMetadataResolver; +import org.elasticsearch.client.Node.Roles; import org.junit.After; import org.junit.Before; import org.mockito.invocation.InvocationOnMock; @@ -67,8 +67,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static java.util.Collections.singletonMap; - /** * Tests for {@link RestClient} behaviour against multiple hosts: fail-over, blacklisting etc. * Relies on a mock http client to intercept requests and return desired responses based on request path. @@ -76,9 +74,8 @@ public class RestClientMultipleHostsTests extends RestClientTestCase { private ExecutorService exec = Executors.newFixedThreadPool(1); - private volatile Map hostMetadata = Collections.emptyMap(); private RestClient restClient; - private HttpHost[] httpHosts; + private Node[] nodes; private HostsTrackingFailureListener failureListener; @Before @@ -115,22 +112,13 @@ public void run() { return null; } }); - int numHosts = RandomNumbers.randomIntBetween(getRandom(), 2, 5); - httpHosts = new HttpHost[numHosts]; - for (int i = 0; i < numHosts; i++) { - httpHosts[i] = new HttpHost("localhost", 9200 + i); + int numNodes = RandomNumbers.randomIntBetween(getRandom(), 2, 5); + nodes = new Node[numNodes]; + for (int i = 0; i < numNodes; i++) { + nodes[i] = new Node(new HttpHost("localhost", 9200 + i)); } - /* - * Back the metadata to a map that we can manipulate during testing. - */ - HostMetadataResolver metaResolver = new HostMetadataResolver() { - @Override - public HostMetadata resolveMetadata(HttpHost host) { - return hostMetadata.get(host); - } - }; failureListener = new HostsTrackingFailureListener(); - restClient = new RestClient(httpClient, 10000, new Header[0], httpHosts, metaResolver, null, failureListener); + restClient = new RestClient(httpClient, 10000, new Header[0], nodes, null, failureListener); } /** @@ -144,9 +132,8 @@ public void shutdownExec() { public void testRoundRobinOkStatusCodes() throws IOException { int numIters = RandomNumbers.randomIntBetween(getRandom(), 1, 5); for (int i = 0; i < numIters; i++) { - Set hostsSet = new HashSet<>(); - Collections.addAll(hostsSet, httpHosts); - for (int j = 0; j < httpHosts.length; j++) { + Set hostsSet = hostsSet(); + for (int j = 0; j < nodes.length; j++) { int statusCode = randomOkStatusCode(getRandom()); Response response = restClient.performRequest(randomHttpMethod(getRandom()), "/" + statusCode); assertEquals(statusCode, response.getStatusLine().getStatusCode()); @@ -160,9 +147,8 @@ public void testRoundRobinOkStatusCodes() throws IOException { public void testRoundRobinNoRetryErrors() throws IOException { int numIters = RandomNumbers.randomIntBetween(getRandom(), 1, 5); for (int i = 0; i < numIters; i++) { - Set hostsSet = new HashSet<>(); - Collections.addAll(hostsSet, httpHosts); - for (int j = 0; j < httpHosts.length; j++) { + Set hostsSet = hostsSet(); + for (int j = 0; j < nodes.length; j++) { String method = randomHttpMethod(getRandom()); int statusCode = randomErrorNoRetryStatusCode(getRandom()); try { @@ -201,10 +187,9 @@ public void testRoundRobinRetryErrors() throws IOException { * the caller. It wraps the exception that contains the failed hosts. */ e = (ResponseException) e.getCause(); - Set hostsSet = new HashSet<>(); - Collections.addAll(hostsSet, httpHosts); + Set hostsSet = hostsSet(); //first request causes all the hosts to be blacklisted, the returned exception holds one suppressed exception each - failureListener.assertCalled(httpHosts); + failureListener.assertCalled(nodes); do { Response response = e.getResponse(); assertEquals(Integer.parseInt(retryEndpoint.substring(1)), response.getStatusLine().getStatusCode()); @@ -226,10 +211,9 @@ public void testRoundRobinRetryErrors() throws IOException { * the caller. It wraps the exception that contains the failed hosts. */ e = (IOException) e.getCause(); - Set hostsSet = new HashSet<>(); - Collections.addAll(hostsSet, httpHosts); + Set hostsSet = hostsSet(); //first request causes all the hosts to be blacklisted, the returned exception holds one suppressed exception each - failureListener.assertCalled(httpHosts); + failureListener.assertCalled(nodes); do { HttpHost httpHost = HttpHost.create(e.getMessage()); assertTrue("host [" + httpHost + "] not found, most likely used multiple times", hostsSet.remove(httpHost)); @@ -248,9 +232,8 @@ public void testRoundRobinRetryErrors() throws IOException { int numIters = RandomNumbers.randomIntBetween(getRandom(), 2, 5); for (int i = 1; i <= numIters; i++) { //check that one different host is resurrected at each new attempt - Set hostsSet = new HashSet<>(); - Collections.addAll(hostsSet, httpHosts); - for (int j = 0; j < httpHosts.length; j++) { + Set hostsSet = hostsSet(); + for (int j = 0; j < nodes.length; j++) { retryEndpoint = randomErrorRetryEndpoint(); try { restClient.performRequest(randomHttpMethod(getRandom()), retryEndpoint); @@ -324,52 +307,30 @@ public void testRoundRobinRetryErrors() throws IOException { } } - /** - * Test that calling {@link RestClient#setHosts(Iterable, HostMetadataResolver)} - * sets the {@link HostMetadataResolver}. - */ - public void testSetHostsWithMetadataResolver() throws IOException { - HostMetadataResolver firstPositionIsClient = new HostMetadataResolver() { + public void testWithNodeSelector() throws IOException { + NodeSelector firstPositionOnly = new NodeSelector() { @Override - public HostMetadata resolveMetadata(HttpHost host) { - HostMetadata.Roles roles; - if (host == httpHosts[0]) { - roles = new HostMetadata.Roles(false, false, false); - } else { - roles = new HostMetadata.Roles(true, true, true); - } - return new HostMetadata("dummy", roles); + public boolean select(Node node) { + return nodes[0] == node; } }; - restClient.setHosts(Arrays.asList(httpHosts), firstPositionIsClient); - assertSame(firstPositionIsClient, restClient.getHostMetadataResolver()); - Response response = restClient.withHostSelector(HostSelector.NOT_MASTER).performRequest("GET", "/200"); - assertEquals(httpHosts[0], response.getHost()); - } - - public void testWithHostSelector() throws IOException { - HostSelector firstPositionOnly = new HostSelector() { - @Override - public boolean select(HttpHost host, HostMetadata meta) { - return httpHosts[0] == host; - } - }; - RestClientActions withHostSelector = restClient.withHostSelector(firstPositionOnly); - Response response = withHostSelector.performRequest("GET", "/200"); - assertEquals(httpHosts[0], response.getHost()); + RestClientActions withNodeSelector = restClient.withNodeSelector(firstPositionOnly); + Response response = withNodeSelector.performRequest("GET", "/200"); + assertEquals(nodes[0].getHost(), response.getHost()); restClient.close(); } - /** - * Test that calling {@link RestClient#setHosts(Iterable)} preserves the - * {@link HostMetadataResolver}. - */ - public void testSetHostsWithoutMetadataResolver() throws IOException { - HttpHost expected = randomFrom(httpHosts); - hostMetadata = singletonMap(expected, new HostMetadata("dummy", new HostMetadata.Roles(false, false, false))); - restClient.setHosts(httpHosts); - Response response = restClient.withHostSelector(HostSelector.NOT_MASTER).performRequest("GET", "/200"); - assertEquals(expected, response.getHost()); + public void testSetNodes() throws IOException { + Node[] newNodes = new Node[nodes.length]; + for (int i = 0; i < nodes.length; i++) { + Roles roles = i == 0 ? new Roles(false, true, true) : new Roles(true, false, false); + newNodes[i] = new Node(nodes[i].getHost(), null, null, roles); + } + restClient.setNodes(newNodes); + Response response = restClient + .withNodeSelector(NodeSelector.NOT_MASTER_ONLY) + .performRequest("GET", "/200"); + assertEquals(newNodes[0].getHost(), response.getHost()); } private static String randomErrorRetryEndpoint() { @@ -385,4 +346,16 @@ private static String randomErrorRetryEndpoint() { } throw new UnsupportedOperationException(); } + + /** + * Build a mutable {@link Set} containing all the {@link Node#getHost() hosts} + * in use by the test. + */ + private Set hostsSet() { + Set hosts = new HashSet<>(); + for (Node node : nodes) { + hosts.add(node.getHost()); + } + return hosts; + } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java index 50a701dc3f56a..2147acf65810c 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java @@ -47,7 +47,6 @@ import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; import org.apache.http.util.EntityUtils; -import org.elasticsearch.client.HostMetadata.HostMetadataResolver; import org.junit.After; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -96,8 +95,7 @@ public class RestClientSingleHostTests extends RestClientTestCase { private ExecutorService exec = Executors.newFixedThreadPool(1); private RestClient restClient; private Header[] defaultHeaders; - private HttpHost httpHost; - private volatile HostMetadata hostMetadata; + private Node node; private CloseableHttpAsyncClient httpClient; private HostsTrackingFailureListener failureListener; @@ -111,7 +109,7 @@ public void createRestClient() throws IOException { public Future answer(InvocationOnMock invocationOnMock) throws Throwable { HttpAsyncRequestProducer requestProducer = (HttpAsyncRequestProducer) invocationOnMock.getArguments()[0]; HttpClientContext context = (HttpClientContext) invocationOnMock.getArguments()[2]; - assertThat(context.getAuthCache().get(httpHost), instanceOf(BasicScheme.class)); + assertThat(context.getAuthCache().get(node.getHost()), instanceOf(BasicScheme.class)); final FutureCallback futureCallback = (FutureCallback) invocationOnMock.getArguments()[3]; HttpUriRequest request = (HttpUriRequest)requestProducer.generateRequest(); @@ -149,16 +147,10 @@ public void run() { }); defaultHeaders = RestClientTestUtil.randomHeaders(getRandom(), "Header-default"); - httpHost = new HttpHost("localhost", 9200); + node = new Node(new HttpHost("localhost", 9200)); failureListener = new HostsTrackingFailureListener(); - HostMetadataResolver metaResolver = new HostMetadataResolver(){ - @Override - public HostMetadata resolveMetadata(HttpHost host) { - return hostMetadata; - } - }; - restClient = new RestClient(httpClient, 10000, defaultHeaders, new HttpHost[]{httpHost}, - metaResolver, null, failureListener); + restClient = new RestClient(httpClient, 10000, defaultHeaders, new Node[] {node}, + null, failureListener); } /** @@ -210,37 +202,55 @@ public void testSetHostsFailures() throws IOException { restClient.setHosts((HttpHost[]) null); fail("setHosts should have failed"); } catch (IllegalArgumentException e) { - assertEquals("hosts must not be null", e.getMessage()); + assertEquals("hosts must not be null or empty", e.getMessage()); } try { restClient.setHosts(); fail("setHosts should have failed"); } catch (IllegalArgumentException e) { - assertEquals("hosts must not be empty", e.getMessage()); + assertEquals("hosts must not be null or empty", e.getMessage()); } try { restClient.setHosts((HttpHost) null); fail("setHosts should have failed"); - } catch (NullPointerException e) { + } catch (IllegalArgumentException e) { assertEquals("host cannot be null", e.getMessage()); } try { restClient.setHosts(new HttpHost("localhost", 9200), null, new HttpHost("localhost", 9201)); fail("setHosts should have failed"); - } catch (NullPointerException e) { + } catch (IllegalArgumentException e) { assertEquals("host cannot be null", e.getMessage()); } + } + + public void testSetNodesFailures() throws IOException { try { - restClient.setHosts(null, HostMetadata.EMPTY_RESOLVER); - fail("setHosts should have failed"); + restClient.setNodes((Node[]) null); + fail("setNodes should have failed"); } catch (IllegalArgumentException e) { - assertEquals("hosts must not be null", e.getMessage()); + assertEquals("nodes must not be null or empty", e.getMessage()); } try { - restClient.setHosts(Arrays.asList(new HttpHost("localhost", 9200)), null); - fail("setHosts should have failed"); + restClient.setNodes(); + fail("setNodes should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("nodes must not be null or empty", e.getMessage()); + } + try { + restClient.setNodes((Node) null); + fail("setNodes should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("node cannot be null", e.getMessage()); + } + try { + restClient.setNodes( + new Node(new HttpHost("localhost", 9200)), + null, + new Node(new HttpHost("localhost", 9201))); + fail("setNodes should have failed"); } catch (IllegalArgumentException e) { - assertEquals("metaResolver must not be null", e.getMessage()); + assertEquals("node cannot be null", e.getMessage()); } } @@ -304,7 +314,7 @@ public void testErrorStatusCodes() throws IOException { if (errorStatusCode <= 500 || expectedIgnores.contains(errorStatusCode)) { failureListener.assertNotCalled(); } else { - failureListener.assertCalled(httpHost); + failureListener.assertCalled(node); } } } @@ -319,14 +329,14 @@ public void testIOExceptions() throws IOException { } catch(IOException e) { assertThat(e, instanceOf(ConnectTimeoutException.class)); } - failureListener.assertCalled(httpHost); + failureListener.assertCalled(node); try { performRequest(method, "/soe"); fail("request should have failed"); } catch(IOException e) { assertThat(e, instanceOf(SocketTimeoutException.class)); } - failureListener.assertCalled(httpHost); + failureListener.assertCalled(node); } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java index 7648b3023d5e6..3fae90625af64 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java @@ -22,9 +22,8 @@ import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; -import org.elasticsearch.client.HostMetadata.HostMetadataResolver; -import org.elasticsearch.client.RestClient.HostTuple; import org.elasticsearch.client.RestClient.NextHostsResult; +import org.elasticsearch.client.RestClient.NodeTuple; import java.io.IOException; import java.net.URI; @@ -51,10 +50,10 @@ public class RestClientTests extends RestClientTestCase { public void testCloseIsIdempotent() throws IOException { - HttpHost[] hosts = new HttpHost[]{new HttpHost("localhost", 9200)}; + Node[] nodes = new Node[] {new Node(new HttpHost("localhost", 9200))}; CloseableHttpAsyncClient closeableHttpAsyncClient = mock(CloseableHttpAsyncClient.class); RestClient restClient = new RestClient(closeableHttpAsyncClient, 1_000, new Header[0], - hosts, HostMetadata.EMPTY_RESOLVER, null, null); + nodes, null, null); restClient.close(); verify(closeableHttpAsyncClient, times(1)).close(); restClient.close(); @@ -165,50 +164,44 @@ public void testNextHostsOneTime() { HttpHost h1 = new HttpHost("1"); HttpHost h2 = new HttpHost("2"); HttpHost h3 = new HttpHost("3"); - Set hosts = new HashSet<>(); - hosts.add(h1); - hosts.add(h2); - hosts.add(h3); + Set nodes = new HashSet<>(); + nodes.add(new Node(h1, null, "1", null)); + nodes.add(new Node(h2, null, "2", null)); + nodes.add(new Node(h3, null, "3", null)); - HostMetadataResolver versionIsName = new HostMetadataResolver() { + NodeSelector not1 = new NodeSelector() { @Override - public HostMetadata resolveMetadata(HttpHost host) { - return new HostMetadata(host.toHostString(), new HostMetadata.Roles(true, true, true)); + public boolean select(Node node) { + return false == "1".equals(node.getVersion()); } }; - HostSelector not1 = new HostSelector() { + NodeSelector noNodes = new NodeSelector() { @Override - public boolean select(HttpHost host, HostMetadata meta) { - return false == "1".equals(meta.version()); - } - }; - HostSelector noHosts = new HostSelector() { - @Override - public boolean select(HttpHost host, HostMetadata meta) { + public boolean select(Node node) { return false; } }; - HostTuple> hostTuple = new HostTuple<>(hosts, null, versionIsName); + NodeTuple> nodeTuple = new NodeTuple<>(nodes, null); Map blacklist = new HashMap<>(); - AtomicInteger lastHostIndex = new AtomicInteger(0); + AtomicInteger lastNodeIndex = new AtomicInteger(0); long now = 0; // Normal case - NextHostsResult result = RestClient.nextHostsOneTime(hostTuple, blacklist, - lastHostIndex, now, HostSelector.ANY); + NextHostsResult result = RestClient.nextHostsOneTime(nodeTuple, blacklist, + lastNodeIndex, now, NodeSelector.ANY); assertThat(result.hosts, containsInAnyOrder(h1, h2, h3)); List expectedHosts = new ArrayList<>(result.hosts); // Calling it again rotates the set of results for (int i = 0; i < iterations; i++) { Collections.rotate(expectedHosts, 1); - assertEquals(expectedHosts, RestClient.nextHostsOneTime(hostTuple, blacklist, - lastHostIndex, now, HostSelector.ANY).hosts); + assertEquals(expectedHosts, RestClient.nextHostsOneTime(nodeTuple, blacklist, + lastNodeIndex, now, NodeSelector.ANY).hosts); } - // Exclude some host - lastHostIndex.set(0); - result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, now, not1); + // Exclude some node + lastNodeIndex.set(0); + result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, now, not1); assertThat(result.hosts, containsInAnyOrder(h2, h3)); // h1 excluded assertEquals(0, result.blacklisted); assertEquals(1, result.selectorRejected); @@ -217,31 +210,31 @@ public boolean select(HttpHost host, HostMetadata meta) { // Calling it again rotates the set of results for (int i = 0; i < iterations; i++) { Collections.rotate(expectedHosts, 1); - assertEquals(expectedHosts, RestClient.nextHostsOneTime(hostTuple, blacklist, - lastHostIndex, now, not1).hosts); + assertEquals(expectedHosts, RestClient.nextHostsOneTime(nodeTuple, blacklist, + lastNodeIndex, now, not1).hosts); } /* - * Try a HostSelector that excludes all hosts. This should + * Try a NodeSelector that excludes all nodes. This should * return a failure. */ - result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, now, noHosts); + result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, now, noNodes); assertNull(result.hosts); assertEquals(0, result.blacklisted); assertEquals(3, result.selectorRejected); assertEquals(0, result.selectorBlockedRevival); /* - * Mark all hosts as dead and look up at a time *after* the - * revival time. This should return all hosts. + * Mark all nodes as dead and look up at a time *after* the + * revival time. This should return all nodes. */ blacklist.put(h1, new DeadHostState(1, 1)); blacklist.put(h2, new DeadHostState(1, 2)); blacklist.put(h3, new DeadHostState(1, 3)); - lastHostIndex.set(0); + lastNodeIndex.set(0); now = 1000; - result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, - now, HostSelector.ANY); + result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, + now, NodeSelector.ANY); assertThat(result.hosts, containsInAnyOrder(h1, h2, h3)); assertEquals(0, result.blacklisted); assertEquals(0, result.selectorRejected); @@ -250,28 +243,28 @@ public boolean select(HttpHost host, HostMetadata meta) { // Calling it again rotates the set of results for (int i = 0; i < iterations; i++) { Collections.rotate(expectedHosts, 1); - assertEquals(expectedHosts, RestClient.nextHostsOneTime(hostTuple, blacklist, - lastHostIndex, now, HostSelector.ANY).hosts); + assertEquals(expectedHosts, RestClient.nextHostsOneTime(nodeTuple, blacklist, + lastNodeIndex, now, NodeSelector.ANY).hosts); } /* - * Now try with the hosts dead and *not* past their dead time. - * Only the host closest to revival should come back. + * Now try with the nodes dead and *not* past their dead time. + * Only the node closest to revival should come back. */ now = 0; - result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, - now, HostSelector.ANY); + result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, + now, NodeSelector.ANY); assertEquals(Collections.singleton(h1), result.hosts); assertEquals(3, result.blacklisted); assertEquals(0, result.selectorRejected); assertEquals(0, result.selectorBlockedRevival); /* - * Now try with the hosts dead and *not* past their dead time - * *and* a host selector that removes the host that is closest - * to being revived. The second closest host should come back. + * Now try with the nodes dead and *not* past their dead time + * *and* a node selector that removes the node that is closest + * to being revived. The second closest node should come back. */ - result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, + result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, now, not1); assertEquals(Collections.singleton(h2), result.hosts); assertEquals(3, result.blacklisted); @@ -279,12 +272,12 @@ public boolean select(HttpHost host, HostMetadata meta) { assertEquals(1, result.selectorBlockedRevival); /* - * Try a HostSelector that excludes all hosts. This should + * Try a NodeSelector that excludes all nodes. This should * return a failure, but a different failure than normal * because it'll block revival rather than outright reject - * healthy hosts. + * healthy nodes. */ - result = RestClient.nextHostsOneTime(hostTuple, blacklist, lastHostIndex, now, noHosts); + result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, now, noNodes); assertNull(result.hosts); assertEquals(3, result.blacklisted); assertEquals(0, result.selectorRejected); @@ -292,8 +285,10 @@ public boolean select(HttpHost host, HostMetadata meta) { } private static RestClient createRestClient() { - HttpHost[] hosts = new HttpHost[]{new HttpHost("localhost", 9200)}; + Node[] hosts = new Node[] { + new Node(new HttpHost("localhost", 9200)) + }; return new RestClient(mock(CloseableHttpAsyncClient.class), randomLongBetween(1_000, 30_000), - new Header[] {}, hosts, HostMetadata.EMPTY_RESOLVER, null, null); + new Header[] {}, hosts, null, null); } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java index dd4939b4e35cf..bdf977c943588 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java +++ b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java @@ -37,7 +37,7 @@ import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.http.util.EntityUtils; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.client.HttpAsyncResponseConsumerFactory; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseListener; @@ -258,11 +258,13 @@ public void onFailure(Exception exception) { } } - public void testHostSelector() throws IOException { + @SuppressWarnings("unused") + public void testNodeSelector() throws IOException { + // TODO link me to docs try (RestClient restClient = RestClient.builder( new HttpHost("localhost", 9200, "http"), new HttpHost("localhost", 9201, "http")).build()) { - RestClientActions client = restClient.withHostSelector(HostSelector.NOT_MASTER); + RestClientActions client = restClient.withNodeSelector(NodeSelector.NOT_MASTER_ONLY); client.performRequest("POST", "/test_index/test_type", Collections.emptyMap(), new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON)); } diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java index d3f964498590a..7a6e4d3493977 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java @@ -26,10 +26,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; -import org.elasticsearch.client.HostMetadata; +import org.elasticsearch.client.Node; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.HostMetadata.Roles; +import org.elasticsearch.client.Node.Roles; import java.io.IOException; import java.io.InputStream; @@ -93,19 +93,18 @@ public ElasticsearchHostsSniffer(RestClient restClient, long sniffRequestTimeout * Calls the elasticsearch nodes info api, parses the response and returns all the found http hosts */ @Override - public SnifferResult sniffHosts() throws IOException { + public List sniffHosts() throws IOException { Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams); return readHosts(response.getEntity()); } - private SnifferResult readHosts(HttpEntity entity) throws IOException { + private List readHosts(HttpEntity entity) throws IOException { try (InputStream inputStream = entity.getContent()) { JsonParser parser = jsonFactory.createParser(inputStream); if (parser.nextToken() != JsonToken.START_OBJECT) { throw new IOException("expected data to start with an object"); } - List hosts = new ArrayList<>(); - Map hostMetadata = new HashMap<>(); + List nodes = new ArrayList<>(); while (parser.nextToken() != JsonToken.END_OBJECT) { if (parser.getCurrentToken() == JsonToken.START_OBJECT) { if ("nodes".equals(parser.getCurrentName())) { @@ -113,19 +112,18 @@ private SnifferResult readHosts(HttpEntity entity) throws IOException { JsonToken token = parser.nextToken(); assert token == JsonToken.START_OBJECT; String nodeId = parser.getCurrentName(); - readHost(nodeId, parser, scheme, hosts, hostMetadata); + readHost(nodeId, parser, scheme, nodes); } } else { parser.skipChildren(); } } } - return new SnifferResult(hosts, hostMetadata); + return nodes; } } - private static void readHost(String nodeId, JsonParser parser, Scheme scheme, List hosts, - Map hostMetadata) throws IOException { + private static void readHost(String nodeId, JsonParser parser, Scheme scheme, List nodes) throws IOException { HttpHost publishedHost = null; List boundHosts = new ArrayList<>(); String fieldName = null; @@ -190,11 +188,9 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Li } else { logger.trace("adding node [" + nodeId + "]"); assert sawRoles : "didn't see roles for [" + nodeId + "]"; - hosts.add(publishedHost); - HostMetadata meta = new HostMetadata(version, new Roles(master, data, ingest)); - for (HttpHost bound: boundHosts) { - hostMetadata.put(bound, meta); - } + assert boundHosts.contains(publishedHost) : + "[" + nodeId + "] doesn't make sense! publishedHost should be in boundHosts"; + nodes.add(new Node(publishedHost, boundHosts, version, new Roles(master, data, ingest))); } } diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java index 99865ea548c8f..9de8134208ec1 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/HostsSniffer.java @@ -20,8 +20,10 @@ package org.elasticsearch.client.sniff; import org.apache.http.HttpHost; +import org.elasticsearch.client.Node; import java.io.IOException; +import java.util.List; import java.util.Map; /** @@ -32,5 +34,5 @@ public interface HostsSniffer { * Returns a {@link Map} from sniffed {@link HttpHost} to metadata * sniffed about the host. */ - SnifferResult sniffHosts() throws IOException; + List sniffHosts() throws IOException; } diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java index 37960d99454e1..95332a90957ad 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/Sniffer.java @@ -22,18 +22,16 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpHost; -import org.elasticsearch.client.HostMetadata; +import org.elasticsearch.client.Node; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.HostMetadata.HostMetadataResolver; import java.io.Closeable; import java.io.IOException; import java.security.AccessController; import java.security.PrivilegedAction; -import java.util.ArrayList; +import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; @@ -120,22 +118,19 @@ void sniffOnFailure(HttpHost failedHost) { void sniff(HttpHost excludeHost, long nextSniffDelayMillis) { if (running.compareAndSet(false, true)) { try { - final SnifferResult sniffedHosts = hostsSniffer.sniffHosts(); + final List sniffedHosts = hostsSniffer.sniffHosts(); logger.debug("sniffed hosts: " + sniffedHosts); if (excludeHost != null) { - sniffedHosts.hosts().remove(excludeHost); - sniffedHosts.hostMetadata().remove(excludeHost); + for (Iterator itr = sniffedHosts.iterator(); itr.hasNext();) { + if (itr.next().getHost().equals(excludeHost)) { + itr.remove(); + } + } } - if (sniffedHosts.hosts().isEmpty()) { + if (sniffedHosts.isEmpty()) { logger.warn("no hosts to set, hosts will be updated at the next sniffing round"); } else { - HostMetadataResolver resolver = new HostMetadataResolver() { - @Override - public HostMetadata resolveMetadata(HttpHost host) { - return sniffedHosts.hostMetadata().get(host); - } - }; - this.restClient.setHosts(sniffedHosts.hosts(), resolver); + this.restClient.setNodes(sniffedHosts.toArray(new Node[0])); } } catch (Exception e) { logger.error("error while sniffing nodes", e); diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java deleted file mode 100644 index 527ec1fd32152..0000000000000 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/SnifferResult.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.client.sniff; - -import org.apache.http.HttpHost; -import org.elasticsearch.client.HostMetadata; - -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * Result of sniffing hosts. - */ -public final class SnifferResult { - /** - * List of all nodes in the cluster. - */ - private final List hosts; - /** - * Map from each address that each node is listening on to metadata about - * the node. - */ - private final Map hostMetadata; - - public SnifferResult(List hosts, Map hostMetadata) { - this.hosts = Objects.requireNonNull(hosts, "hosts is required"); - this.hostMetadata = Objects.requireNonNull(hostMetadata, "hostMetadata is required"); - } - - /** - * List of all nodes in the cluster. - */ - public List hosts() { - return hosts; - } - - /** - * Map from each address that each node is listening on to metadata about - * the node. - */ - public Map hostMetadata() { - return hostMetadata; - } - - @Override - public boolean equals(Object obj) { - if (obj == null || obj.getClass() != getClass()) { - return false; - } - SnifferResult other = (SnifferResult) obj; - return hosts.equals(other.hosts) - && hostMetadata.equals(other.hostMetadata); - } - - @Override - public int hashCode() { - return Objects.hash(hosts, hostMetadata); - } -} diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java index 74ee74499cafe..ba564f66f5d82 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java @@ -31,7 +31,7 @@ import org.apache.http.HttpHost; import org.apache.http.client.methods.HttpGet; import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement; -import org.elasticsearch.client.HostMetadata; +import org.elasticsearch.client.Node; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; @@ -119,7 +119,7 @@ public void testSniffNodes() throws IOException { try (RestClient restClient = RestClient.builder(httpHost).build()) { ElasticsearchHostsSniffer sniffer = new ElasticsearchHostsSniffer(restClient, sniffRequestTimeout, scheme); try { - SnifferResult result = sniffer.sniffHosts(); + List result = sniffer.sniffHosts(); if (sniffResponse.isFailure) { fail("sniffNodes should have failed"); } @@ -178,8 +178,7 @@ public void handle(HttpExchange httpExchange) throws IOException { private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme scheme) throws IOException { int numNodes = RandomNumbers.randomIntBetween(getRandom(), 1, 5); - List hosts = new ArrayList<>(numNodes); - Map hostMetadata = new HashMap<>(numNodes); + List nodes = new ArrayList<>(numNodes); JsonFactory jsonFactory = new JsonFactory(); StringWriter writer = new StringWriter(); JsonGenerator generator = jsonFactory.createGenerator(writer); @@ -196,9 +195,19 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme String nodeId = RandomStrings.randomAsciiOfLengthBetween(getRandom(), 5, 10); String host = "host" + i; int port = RandomNumbers.randomIntBetween(getRandom(), 9200, 9299); - HttpHost httpHost = new HttpHost(host, port, scheme.toString()); - HostMetadata metadata = new HostMetadata(randomAsciiAlphanumOfLength(5), - new HostMetadata.Roles(randomBoolean(), randomBoolean(), randomBoolean())); + HttpHost publishHost = new HttpHost(host, port, scheme.toString()); + List boundHosts = new ArrayList<>(); + boundHosts.add(publishHost); + + if (randomBoolean()) { + int bound = between(1, 5); + for (int b = 0; b < bound; b++) { + boundHosts.add(new HttpHost(host + b, port, scheme.toString())); + } + } + + Node node = new Node(publishHost, boundHosts, randomAsciiAlphanumOfLength(5), + new Node.Roles(randomBoolean(), randomBoolean(), randomBoolean())); generator.writeObjectFieldStart(nodeId); if (getRandom().nextBoolean()) { @@ -213,25 +222,18 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme } boolean isHttpEnabled = rarely() == false; if (isHttpEnabled) { - hosts.add(httpHost); - hostMetadata.put(httpHost, metadata); + nodes.add(node); generator.writeObjectFieldStart("http"); generator.writeArrayFieldStart("bound_address"); - generator.writeString(httpHost.toHostString()); - if (randomBoolean()) { - int extras = between(1, 5); - for (int e = 0; e < extras; e++) { - HttpHost extraHost = new HttpHost(httpHost.getHostName() + e, port, scheme.toString()); - hostMetadata.put(extraHost, metadata); - generator.writeString(extraHost.toHostString()); - } + for (HttpHost bound : boundHosts) { + generator.writeString(bound.toHostString()); } generator.writeEndArray(); if (getRandom().nextBoolean()) { generator.writeObjectFieldStart("bogus_object"); generator.writeEndObject(); } - generator.writeStringField("publish_address", httpHost.toHostString()); + generator.writeStringField("publish_address", publishHost.toHostString()); if (getRandom().nextBoolean()) { generator.writeNumberField("max_content_length_in_bytes", 104857600); } @@ -242,20 +244,20 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme Collections.shuffle(roles, getRandom()); generator.writeArrayFieldStart("roles"); for (String role : roles) { - if ("master".equals(role) && metadata.roles().master()) { + if ("master".equals(role) && node.getRoles().hasMaster()) { generator.writeString("master"); } - if ("data".equals(role) && metadata.roles().data()) { + if ("data".equals(role) && node.getRoles().hasData()) { generator.writeString("data"); } - if ("ingest".equals(role) && metadata.roles().ingest()) { + if ("ingest".equals(role) && node.getRoles().hasIngest()) { generator.writeString("ingest"); } } generator.writeEndArray(); generator.writeFieldName("version"); - generator.writeString(metadata.version()); + generator.writeString(node.getVersion()); int numAttributes = RandomNumbers.randomIntBetween(getRandom(), 0, 3); Map attributes = new HashMap<>(numAttributes); @@ -276,16 +278,16 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme generator.writeEndObject(); generator.writeEndObject(); generator.close(); - return SniffResponse.buildResponse(writer.toString(), new SnifferResult(hosts, hostMetadata)); + return SniffResponse.buildResponse(writer.toString(), nodes); } private static class SniffResponse { private final String nodesInfoBody; private final int nodesInfoResponseCode; - private final SnifferResult result; + private final List result; private final boolean isFailure; - SniffResponse(String nodesInfoBody, SnifferResult result, boolean isFailure) { + SniffResponse(String nodesInfoBody, List result, boolean isFailure) { this.nodesInfoBody = nodesInfoBody; this.result = result; this.isFailure = isFailure; @@ -300,7 +302,7 @@ static SniffResponse buildFailure() { return new SniffResponse("", null, true); } - static SniffResponse buildResponse(String nodesInfoBody, SnifferResult result) { + static SniffResponse buildResponse(String nodesInfoBody, List result) { return new SniffResponse(nodesInfoBody, result, false); } } diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java index 40aab7228a80b..6bad7ee1af6ee 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java @@ -20,21 +20,20 @@ package org.elasticsearch.client.sniff; import org.apache.http.HttpHost; -import org.elasticsearch.client.HostMetadata; +import org.elasticsearch.client.Node; import java.io.IOException; import java.util.Collections; -import java.util.Map; +import java.util.List; /** * Mock implementation of {@link HostsSniffer}. Useful to prevent any connection attempt while testing builders etc. */ class MockHostsSniffer implements HostsSniffer { @Override - public SnifferResult sniffHosts() throws IOException { - HttpHost host = new HttpHost("localhost", 9200); - return new SnifferResult(Collections.singletonList(host), - Collections.singletonMap(new HttpHost("localhost", 9200), - new HostMetadata("mock version", new HostMetadata.Roles(false, false, false)))); + public List sniffHosts() throws IOException { + return Collections.singletonList(new Node( + new HttpHost("localhost", 9200), Collections.emptyList(), + "mock version", new Node.Roles(false, false, false))); } } diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java index 91699be46f5fd..541f3822f7505 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/documentation/SnifferDocumentation.java @@ -20,13 +20,12 @@ package org.elasticsearch.client.sniff.documentation; import org.apache.http.HttpHost; -import org.elasticsearch.client.HostMetadata; +import org.elasticsearch.client.Node; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer; import org.elasticsearch.client.sniff.HostsSniffer; import org.elasticsearch.client.sniff.SniffOnFailureListener; import org.elasticsearch.client.sniff.Sniffer; -import org.elasticsearch.client.sniff.SnifferResult; import java.io.IOException; import java.util.List; @@ -122,7 +121,7 @@ public void testUsage() throws IOException { .build(); HostsSniffer hostsSniffer = new HostsSniffer() { @Override - public SnifferResult sniffHosts() throws IOException { + public List sniffHosts() throws IOException { return null; // <1> } }; diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/10_with_id.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/10_with_id.yml index daac81849fb5e..056a4aec1dfd4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/10_with_id.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/10_with_id.yml @@ -1,7 +1,11 @@ --- "Index with ID": - + # NOCOMMIT check that everything is wired up. This should fail if run. + - skip: + features: node_selector - do: + node_selector: + version: " - 6.9.99" index: index: test-weird-index-中文 type: weird.type diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 108ecc1be3a90..b92dcf1ba3d56 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -32,8 +32,7 @@ import org.apache.http.ssl.SSLContexts; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.HostMetadata; +import org.elasticsearch.client.Node; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; @@ -68,13 +67,16 @@ import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -542,29 +544,61 @@ protected static Map getAsMap(final String endpoint) throws IOEx * same thing as using the {@link Sniffer} because: *

    *
  • It doesn't replace the hosts that that {@link #client} communicates with - *
  • It only runs once + *
  • If there is already host metadata it skips running. This behavior isn't + * thread safe but it doesn't have to be for our tests. *
*/ protected void sniffHostMetadata(RestClient client) throws IOException { - if (HostMetadata.EMPTY_RESOLVER != client.getHostMetadataResolver()) { - // Already added a resolver + Node[] nodes = client.getNodes(); + boolean allHaveRoles = true; + for (Node node : nodes) { + if (node.getRoles() == null) { + allHaveRoles = false; + break; + } + } + if (allHaveRoles) { + // We already have resolved metadata. return; } // No resolver, sniff one time and resolve metadata against the results - ElasticsearchHostsSniffer.Scheme scheme; - switch (getProtocol()) { - case "http": - scheme = ElasticsearchHostsSniffer.Scheme.HTTP; - break; - case "https": - scheme = ElasticsearchHostsSniffer.Scheme.HTTPS; - break; - default: - throw new UnsupportedOperationException("unknown protocol [" + getProtocol() + "]"); - } + ElasticsearchHostsSniffer.Scheme scheme = + ElasticsearchHostsSniffer.Scheme.valueOf(getProtocol().toUpperCase(Locale.ROOT)); + /* + * We don't want to change the list of nodes that the client communicates with + * because that'd just be rude. So instead we replace the nodes with nodes the + * that + */ ElasticsearchHostsSniffer sniffer = new ElasticsearchHostsSniffer( - adminClient, ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT, scheme); - Map meta = sniffer.sniffHosts().hostMetadata(); - client.setHosts(clusterHosts, meta::get); + adminClient, ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT, scheme); + attachSniffedMetadataOnClient(client, nodes, sniffer.sniffHosts()); + } + + static void attachSniffedMetadataOnClient(RestClient client, Node[] originalNodes, List nodesWithMetadata) { + Set originalHosts = Arrays.stream(originalNodes) + .map(Node::getHost) + .collect(Collectors.toSet()); + List sniffed = new ArrayList<>(); + for (Node node : nodesWithMetadata) { + if (originalHosts.contains(node.getHost())) { + sniffed.add(node); + } else { + for (HttpHost bound : node.getBoundHosts()) { + if (originalHosts.contains(bound)) { + sniffed.add(node.withBoundHostAsHost(bound)); + break; + } + } + } + } + int missing = originalNodes.length - sniffed.size(); + if (missing > 0) { + List hosts = Arrays.stream(originalNodes) + .map(Node::getHost) + .collect(Collectors.toList()); + throw new IllegalStateException("Didn't sniff metadata for all nodes. Wanted metadata for " + + hosts + " but got " + sniffed); + } + client.setNodes(sniffed.toArray(new Node[0])); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java index 74a8479bfdfa4..4ef45ed5db1da 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlDocsTestClient.java @@ -22,7 +22,7 @@ import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.elasticsearch.Version; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; @@ -49,7 +49,7 @@ public ClientYamlDocsTestClient(ClientYamlSuiteRestSpec restSpec, RestClient res @Override public ClientYamlTestResponse callApi(String apiName, Map params, HttpEntity entity, - Map headers, HostSelector hostSelector) throws IOException { + Map headers, NodeSelector nodeSelector) throws IOException { if ("raw".equals(apiName)) { // Raw requests are bit simpler.... Map queryStringParams = new HashMap<>(params); @@ -63,6 +63,6 @@ public ClientYamlTestResponse callApi(String apiName, Map params throw new ClientYamlTestResponseException(e); } } - return super.callApi(apiName, params, entity, headers, hostSelector); + return super.callApi(apiName, params, entity, headers, nodeSelector); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java index 36e09364badd1..89fc3843a72ec 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestClient.java @@ -28,7 +28,7 @@ import org.apache.http.util.EntityUtils; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; @@ -78,7 +78,7 @@ public Version getEsVersion() { * Calls an api with the provided parameters and body */ public ClientYamlTestResponse callApi(String apiName, Map params, HttpEntity entity, - Map headers, HostSelector hostSelector) throws IOException { + Map headers, NodeSelector nodeSelector) throws IOException { ClientYamlSuiteRestApi restApi = restApi(apiName); @@ -173,7 +173,7 @@ public ClientYamlTestResponse callApi(String apiName, Map params logger.debug("calling api [{}]", apiName); try { Response response = restClient - .withHostSelector(hostSelector) + .withNodeSelector(nodeSelector) .performRequest(requestMethod, requestPath, queryStringParams, entity, requestHeaders); return new ClientYamlTestResponse(response); } catch(ResponseException e) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java index a7b00bea86289..1556fa0b88c49 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContext.java @@ -25,7 +25,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.CheckedRunnable; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.Loggers; @@ -59,13 +59,13 @@ public class ClientYamlTestExecutionContext { private final boolean randomizeContentType; - private final CheckedRunnable setHostMetadata; + private final CheckedRunnable setNodeMetadata; - ClientYamlTestExecutionContext(ClientYamlTestClient clientYamlTestClient, CheckedRunnable setHostMetadata, + ClientYamlTestExecutionContext(ClientYamlTestClient clientYamlTestClient, CheckedRunnable setNodeMetadata, boolean randomizeContentType) { this.clientYamlTestClient = clientYamlTestClient; this.randomizeContentType = randomizeContentType; - this.setHostMetadata = setHostMetadata; + this.setNodeMetadata = setNodeMetadata; } /** @@ -74,7 +74,7 @@ public class ClientYamlTestExecutionContext { */ public ClientYamlTestResponse callApi(String apiName, Map params, List> bodies, Map headers) throws IOException { - return callApi(apiName, params, bodies, headers, HostSelector.ANY); + return callApi(apiName, params, bodies, headers, NodeSelector.ANY); } /** @@ -82,7 +82,7 @@ public ClientYamlTestResponse callApi(String apiName, Map params * Saves the obtained response in the execution context. */ public ClientYamlTestResponse callApi(String apiName, Map params, List> bodies, - Map headers, HostSelector hostSelector) throws IOException { + Map headers, NodeSelector nodeSelector) throws IOException { //makes a copy of the parameters before modifying them for this specific request Map requestParams = new HashMap<>(params); requestParams.putIfAbsent("error_trace", "true"); // By default ask for error traces, this my be overridden by params @@ -100,13 +100,13 @@ public ClientYamlTestResponse callApi(String apiName, Map params } } - if (hostSelector != HostSelector.ANY) { - setHostMetadata.run(); + if (nodeSelector != NodeSelector.ANY) { + setNodeMetadata.run(); } HttpEntity entity = createEntity(bodies, requestHeaders); try { - response = callApiInternal(apiName, requestParams, entity, requestHeaders, hostSelector); + response = callApiInternal(apiName, requestParams, entity, requestHeaders, nodeSelector); return response; } catch(ClientYamlTestResponseException e) { response = e.getRestTestResponse(); @@ -173,8 +173,8 @@ private BytesRef bodyAsBytesRef(Map bodyAsMap, XContentType xCon // pkg-private for testing ClientYamlTestResponse callApiInternal(String apiName, Map params, - HttpEntity entity, Map headers, HostSelector hostSelector) throws IOException { - return clientYamlTestClient.callApi(apiName, params, entity, headers, hostSelector); + HttpEntity entity, Map headers, NodeSelector nodeSelector) throws IOException { + return clientYamlTestClient.callApi(apiName, params, entity, headers, nodeSelector); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java index d2ca61f4d03c1..31fa59857cfe2 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java @@ -39,7 +39,7 @@ public final class Features { "catch_unauthorized", "embedded_stash_key", "headers", - "host_selector", + "node_selector", "stash_in_key", "stash_in_path", "stash_path_replace", diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java index 2efa9220d2264..de73fefaea776 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ApiCallSection.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.Map; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.NodeSelector; import static java.util.Collections.unmodifiableMap; @@ -37,7 +37,7 @@ public class ApiCallSection { private final Map params = new HashMap<>(); private final Map headers = new HashMap<>(); private final List> bodies = new ArrayList<>(); - private HostSelector hostSelector = HostSelector.ANY; + private NodeSelector nodeSelector = NodeSelector.ANY; public ApiCallSection(String api) { this.api = api; @@ -83,14 +83,14 @@ public boolean hasBody() { /** * Selects the node on which to run this request. */ - public HostSelector getHostSelector() { - return hostSelector; + public NodeSelector getNodeSelector() { + return nodeSelector; } /** * Set the selector that decides which node can run this request. */ - public void setHostSelector(HostSelector hostSelector) { - this.hostSelector = hostSelector; + public void setNodeSelector(NodeSelector nodeSelector) { + this.nodeSelector = nodeSelector; } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java index 09d9cb849812a..1ec2382fac596 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSection.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.test.rest.yaml.section; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; @@ -92,10 +92,10 @@ public void addExecutableSection(ExecutableSection executableSection) { + "runners that do not support the [warnings] section can skip the test at line [" + doSection.getLocation().lineNumber + "]"); } - if (HostSelector.ANY != doSection.getApiCallSection().getHostSelector() - && false == skipSection.getFeatures().contains("host_selector")) { - throw new IllegalArgumentException("Attempted to add a [do] with a [host_selector] section without a corresponding " - + "[skip] so runners that do not support the [host_selector] section can skip the test at line [" + if (NodeSelector.ANY != doSection.getApiCallSection().getNodeSelector() + && false == skipSection.getFeatures().contains("node_selector")) { + throw new IllegalArgumentException("Attempted to add a [do] with a [node_selector] section without a corresponding " + + "[skip] so runners that do not support the [node_selector] section can skip the test at line [" + doSection.getLocation().lineNumber + "]"); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index ac491e81a3755..586fbf3e56e8a 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -22,8 +22,8 @@ import org.apache.http.HttpHost; import org.apache.logging.log4j.Logger; import org.elasticsearch.Version; -import org.elasticsearch.client.HostMetadata; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -88,7 +88,7 @@ public static DoSection parse(XContentParser parser) throws IOException { DoSection doSection = new DoSection(parser.getTokenLocation()); ApiCallSection apiCallSection = null; - HostSelector hostSelector = HostSelector.ANY; + NodeSelector nodeSelector = NodeSelector.ANY; Map headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); List expectedWarnings = new ArrayList<>(); @@ -125,26 +125,15 @@ public static DoSection parse(XContentParser parser) throws IOException { headers.put(headerName, parser.text()); } } - } else if ("host_selector".equals(currentFieldName)) { + } else if ("node_selector".equals(currentFieldName)) { String selectorName = null; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { selectorName = parser.currentName(); } else if (token.isValue()) { - final HostSelector original = hostSelector; - final HostSelector newSelector = buildHostSelector( + NodeSelector newSelector = buildNodeSelector( parser.getTokenLocation(), selectorName, parser.text()); - hostSelector = new HostSelector() { - @Override - public boolean select(HttpHost host, HostMetadata meta) { - return original.select(host, meta) && newSelector.select(host, meta); - } - - @Override - public String toString() { - return original + " AND " + newSelector; - } - }; + nodeSelector = new NodeSelector.And(nodeSelector, newSelector); } } } else if (currentFieldName != null) { // must be part of API call then @@ -179,7 +168,7 @@ public String toString() { throw new IllegalArgumentException("client call section is mandatory within a do section"); } apiCallSection.addHeaders(headers); - apiCallSection.setHostSelector(hostSelector); + apiCallSection.setNodeSelector(nodeSelector); doSection.setApiCallSection(apiCallSection); doSection.setExpectedWarningHeaders(unmodifiableList(expectedWarnings)); } finally { @@ -248,7 +237,7 @@ public void execute(ClientYamlTestExecutionContext executionContext) throws IOEx try { ClientYamlTestResponse response = executionContext.callApi(apiCallSection.getApi(), apiCallSection.getParams(), - apiCallSection.getBodies(), apiCallSection.getHeaders(), apiCallSection.getHostSelector()); + apiCallSection.getBodies(), apiCallSection.getHeaders(), apiCallSection.getNodeSelector()); if (Strings.hasLength(catchParam)) { String catchStatusCode; if (catches.containsKey(catchParam)) { @@ -365,17 +354,18 @@ private String formatStatusCodeMessage(ClientYamlTestResponse restTestResponse, not(equalTo(409))))); } - private static HostSelector buildHostSelector(XContentLocation location, String name, String value) { + private static NodeSelector buildNodeSelector(XContentLocation location, String name, String value) { switch (name) { case "version": Version[] range = SkipSection.parseVersionRange(value); - return new HostSelector() { + return new NodeSelector() { @Override - public boolean select(HttpHost host, HostMetadata meta) { - if (meta == null) { - throw new IllegalStateException("expected HostMetadata to be loaded!"); + public boolean select(Node node) { + if (node.getVersion() == null) { + throw new IllegalStateException("expected [version] metadata to be set but got " + + node); } - Version version = Version.fromString(meta.version()); + Version version = Version.fromString(node.getVersion()); return version.onOrAfter(range[0]) && version.onOrBefore(range[1]); } @@ -385,7 +375,7 @@ public String toString() { } }; default: - throw new IllegalArgumentException("unknown host_selector [" + name + "]"); + throw new IllegalArgumentException("unknown node_selector [" + name + "]"); } } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java new file mode 100644 index 0000000000000..2bc6153b34878 --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.rest; + +import org.apache.http.HttpHost; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.List; + +import static java.util.Collections.emptyList; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +public class ESRestTestCaseTests extends ESTestCase { + public void testAttachSniffedMetadataOnClientOk() { + RestClient client = mock(RestClient.class); + Node[] originalNodes = new Node[] { + new Node(new HttpHost("1")), + new Node(new HttpHost("2")), + new Node(new HttpHost("3")), + }; + List nodesWithMetadata = Arrays.asList(new Node[] { + // This node matches exactly: + new Node(new HttpHost("1"), emptyList(), randomAlphaOfLength(5), randomRoles()), + // This node also matches exactly but has bound hosts which don't matter: + new Node(new HttpHost("2"), Arrays.asList(new HttpHost("2"), new HttpHost("not2")), + randomAlphaOfLength(5), randomRoles()), + // This node's host doesn't match but one of its published hosts does so + // we return a modified version of it: + new Node(new HttpHost("not3"), Arrays.asList(new HttpHost("not3"), new HttpHost("3")), + randomAlphaOfLength(5), randomRoles()), + // This node isn't in the original list so it isn't added: + new Node(new HttpHost("4"), emptyList(), randomAlphaOfLength(5), randomRoles()), + }); + ESRestTestCase.attachSniffedMetadataOnClient(client, originalNodes, nodesWithMetadata); + verify(client).setNodes(new Node[] { + nodesWithMetadata.get(0), + nodesWithMetadata.get(1), + nodesWithMetadata.get(2).withBoundHostAsHost(new HttpHost("3")), + }); + } + + public void testAttachSniffedMetadataOnClientNotEnoughNodes() { + // Try a version of the call that should fail because it doesn't have all the results + RestClient client = mock(RestClient.class); + Node[] originalNodes = new Node[] { + new Node(new HttpHost("1")), + new Node(new HttpHost("2")), + }; + List nodesWithMetadata = Arrays.asList(new Node[] { + // This node matches exactly: + new Node(new HttpHost("1"), emptyList(), "v", new Node.Roles(true, true, true)), + }); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> + ESRestTestCase.attachSniffedMetadataOnClient(client, originalNodes, nodesWithMetadata)); + assertEquals(e.getMessage(), "Didn't sniff metadata for all nodes. Wanted metadata for " + + "[http://1, http://2] but got [[host=http://1, bound=[], version=v, roles=mdi]]"); + verify(client, never()).setNodes(any(Node[].class)); + } + + private Node.Roles randomRoles() { + return new Node.Roles(randomBoolean(), randomBoolean(), randomBoolean()); + } +} diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java index c818dec18e51d..c54847088eb7a 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ClientYamlTestExecutionContextTests.java @@ -20,7 +20,7 @@ package org.elasticsearch.test.rest.yaml; import org.apache.http.HttpEntity; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -41,7 +41,7 @@ public void testHeadersSupportStashedValueReplacement() throws IOException { new ClientYamlTestExecutionContext(null, () -> {}, randomBoolean()) { @Override ClientYamlTestResponse callApiInternal(String apiName, Map params, - HttpEntity entity, Map headers, HostSelector hostSelector) { + HttpEntity entity, Map headers, NodeSelector nodeSelector) { headersRef.set(headers); return null; } @@ -62,19 +62,19 @@ ClientYamlTestResponse callApiInternal(String apiName, Map param assertEquals("baz bar1", headersRef.get().get("foo1")); } - public void testNonDefaultHostSelectorSetsHostMetadata() throws IOException { + public void testNonDefaultNodeSelectorSetsNodeMetadata() throws IOException { AtomicBoolean setHostMetadata = new AtomicBoolean(false); final ClientYamlTestExecutionContext context = new ClientYamlTestExecutionContext(null, () -> setHostMetadata.set(true), randomBoolean()) { @Override ClientYamlTestResponse callApiInternal(String apiName, Map params, - HttpEntity entity, Map headers, HostSelector hostSelector) { + HttpEntity entity, Map headers, NodeSelector nodeSelector) { return null; } }; - context.callApi(randomAlphaOfLength(2), emptyMap(), emptyList(), emptyMap(), HostSelector.ANY); + context.callApi(randomAlphaOfLength(2), emptyMap(), emptyList(), emptyMap(), NodeSelector.ANY); assertFalse(setHostMetadata.get()); - context.callApi(randomAlphaOfLength(2), emptyMap(), emptyList(), emptyMap(), HostSelector.NOT_MASTER); + context.callApi(randomAlphaOfLength(2), emptyMap(), emptyList(), emptyMap(), NodeSelector.NOT_MASTER_ONLY); assertTrue(setHostMetadata.get()); } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java index 021c169510b1e..87f2d7f9a53f8 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSectionTests.java @@ -20,7 +20,7 @@ package org.elasticsearch.test.rest.yaml.section; import org.elasticsearch.Version; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.XContentLocation; import org.elasticsearch.common.xcontent.XContentParser; @@ -67,28 +67,28 @@ public void testAddingDoWithWarningWithSkipButNotWarnings() { + " [warnings] section can skip the test at line [" + lineNumber + "]", e.getMessage()); } - public void testAddingDoWithHostSelectorWithSkip() { + public void testAddingDoWithNodeSelectorWithSkip() { int lineNumber = between(1, 10000); ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); - section.setSkipSection(new SkipSection(null, singletonList("host_selector"), null)); + section.setSkipSection(new SkipSection(null, singletonList("node_selector"), null)); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); ApiCallSection apiCall = new ApiCallSection("test"); - apiCall.setHostSelector(HostSelector.NOT_MASTER); + apiCall.setNodeSelector(NodeSelector.NOT_MASTER_ONLY); doSection.setApiCallSection(apiCall); section.addExecutableSection(doSection); } - public void testAddingDoWithHostSelectorWithSkipButNotWarnings() { + public void testAddingDoWithNodeSelectorWithSkipButNotWarnings() { int lineNumber = between(1, 10000); ClientYamlTestSection section = new ClientYamlTestSection(new XContentLocation(0, 0), "test"); section.setSkipSection(new SkipSection(null, singletonList("yaml"), null)); DoSection doSection = new DoSection(new XContentLocation(lineNumber, 0)); ApiCallSection apiCall = new ApiCallSection("test"); - apiCall.setHostSelector(HostSelector.NOT_MASTER); + apiCall.setNodeSelector(NodeSelector.NOT_MASTER_ONLY); doSection.setApiCallSection(apiCall); Exception e = expectThrows(IllegalArgumentException.class, () -> section.addExecutableSection(doSection)); - assertEquals("Attempted to add a [do] with a [host_selector] section without a corresponding" - + " [skip] so runners that do not support the [host_selector] section can skip the test at" + assertEquals("Attempted to add a [do] with a [node_selector] section without a corresponding" + + " [skip] so runners that do not support the [node_selector] section can skip the test at" + " line [" + lineNumber + "]", e.getMessage()); } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java index fe97fecf9985e..6d00ddb7cb08a 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java @@ -20,8 +20,8 @@ package org.elasticsearch.test.rest.yaml.section; import org.apache.http.HttpHost; -import org.elasticsearch.client.HostMetadata; -import org.elasticsearch.client.HostSelector; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.NodeSelector; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.xcontent.XContent; import org.elasticsearch.common.xcontent.XContentLocation; @@ -508,31 +508,33 @@ public void testParseDoSectionExpectedWarnings() throws Exception { "just one entry this time"))); } - public void testHostSelector() throws IOException { + public void testNodeSelector() throws IOException { parser = createParser(YamlXContent.yamlXContent, - "host_selector:\n" + + "node_selector:\n" + " version: 5.2.0-6.0.0\n" + "indices.get_field_mapping:\n" + " index: test_index" ); DoSection doSection = DoSection.parse(parser); - HttpHost host = new HttpHost("dummy"); - HostMetadata.Roles roles = new HostMetadata.Roles(true, true, true); - assertNotSame(HostSelector.ANY, doSection.getApiCallSection().getHostSelector()); - assertTrue(doSection.getApiCallSection().getHostSelector() - .select(host, new HostMetadata("5.2.1", roles))); - assertFalse(doSection.getApiCallSection().getHostSelector() - .select(host, new HostMetadata("6.1.2", roles))); - assertFalse(doSection.getApiCallSection().getHostSelector() - .select(host, new HostMetadata("1.7.0", roles))); + assertNotSame(NodeSelector.ANY, doSection.getApiCallSection().getNodeSelector()); + assertTrue(doSection.getApiCallSection().getNodeSelector() + .select(nodeWithVersion("5.2.1"))); + assertFalse(doSection.getApiCallSection().getNodeSelector() + .select(nodeWithVersion("6.1.2"))); + assertFalse(doSection.getApiCallSection().getNodeSelector() + .select(nodeWithVersion("1.7.0"))); ClientYamlTestExecutionContext context = mock(ClientYamlTestExecutionContext.class); ClientYamlTestResponse mockResponse = mock(ClientYamlTestResponse.class); when(context.callApi("indices.get_field_mapping", singletonMap("index", "test_index"), - emptyList(), emptyMap(), doSection.getApiCallSection().getHostSelector())).thenReturn(mockResponse); + emptyList(), emptyMap(), doSection.getApiCallSection().getNodeSelector())).thenReturn(mockResponse); doSection.execute(context); verify(context).callApi("indices.get_field_mapping", singletonMap("index", "test_index"), - emptyList(), emptyMap(), doSection.getApiCallSection().getHostSelector()); + emptyList(), emptyMap(), doSection.getApiCallSection().getNodeSelector()); + } + + private Node nodeWithVersion(String version) { + return new Node(new HttpHost("dummy"), null, version, null); } private void assertJsonEquals(Map actual, String expected) throws IOException { From 241d7ca14dd9a346af902e62bd96c83d629a2c53 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 26 Mar 2018 17:40:12 -0400 Subject: [PATCH 16/27] Fixup --- .../main/java/org/elasticsearch/client/Node.java | 16 ++++++++-------- .../org/elasticsearch/client/NodeSelector.java | 2 +- .../sniff/ElasticsearchHostsSnifferTests.java | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/Node.java b/client/rest/src/main/java/org/elasticsearch/client/Node.java index 833bc2b05f12e..7954b960835d3 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/Node.java +++ b/client/rest/src/main/java/org/elasticsearch/client/Node.java @@ -147,12 +147,12 @@ public int hashCode() { * Role information about an Elasticsearch process. */ public static final class Roles { - private final boolean master; + private final boolean masterEligible; private final boolean data; private final boolean ingest; - public Roles(boolean master, boolean data, boolean ingest) { - this.master = master; + public Roles(boolean masterEligible, boolean data, boolean ingest) { + this.masterEligible = masterEligible; this.data = data; this.ingest = ingest; } @@ -160,8 +160,8 @@ public Roles(boolean master, boolean data, boolean ingest) { /** * The node could be elected master. */ - public boolean hasMaster() { - return master; + public boolean hasMasterEligible() { + return masterEligible; } /** * The node stores data. @@ -179,7 +179,7 @@ public boolean hasIngest() { @Override public String toString() { StringBuilder result = new StringBuilder(3); - if (master) result.append('m'); + if (masterEligible) result.append('m'); if (data) result.append('d'); if (ingest) result.append('i'); return result.toString(); @@ -191,14 +191,14 @@ public boolean equals(Object obj) { return false; } Roles other = (Roles) obj; - return master == other.master + return masterEligible == other.masterEligible && data == other.data && ingest == other.ingest; } @Override public int hashCode() { - return Objects.hash(master, data, ingest); + return Objects.hash(masterEligible, data, ingest); } } } diff --git a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java index 889a6b2920972..ac97071023b02 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java +++ b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java @@ -56,7 +56,7 @@ public String toString() { @Override public boolean select(Node node) { return node.getRoles() != null - && (false == node.getRoles().hasMaster() || node.getRoles().hasData()); + && (false == node.getRoles().hasMasterEligible() || node.getRoles().hasData()); } @Override diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java index ba564f66f5d82..8a68aab595b4c 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java @@ -244,7 +244,7 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme Collections.shuffle(roles, getRandom()); generator.writeArrayFieldStart("roles"); for (String role : roles) { - if ("master".equals(role) && node.getRoles().hasMaster()) { + if ("master".equals(role) && node.getRoles().hasMasterEligible()) { generator.writeString("master"); } if ("data".equals(role) && node.getRoles().hasData()) { From 84befefe014e80d521e1abcc2cae3900d8160ee3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 27 Mar 2018 09:53:25 -0400 Subject: [PATCH 17/27] Fixup from comments --- .../client/AbstractRestClientActions.java | 8 +++ .../java/org/elasticsearch/client/Node.java | 33 +++++---- .../org/elasticsearch/client/RestClient.java | 2 +- .../client/RestClientActions.java | 12 ++-- .../org/elasticsearch/client/NodeTests.java | 23 ++++++- .../RestClientMultipleHostsIntegTests.java | 68 +++++++++++-------- .../client/RestClientMultipleHostsTests.java | 30 +++++--- 7 files changed, 112 insertions(+), 64 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java index 1579a918196c0..4039194c68694 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/AbstractRestClientActions.java @@ -41,8 +41,16 @@ * {@link RestClient} and {@link RestClientView}. */ abstract class AbstractRestClientActions implements RestClientActions { + /** + * Build a {@link SyncResponseListener} to convert requests from + * asynchronous to synchronous. + */ abstract SyncResponseListener syncResponseListener(); + /** + * Perform the actual request asynchronously, letting any the caller + * handle all exceptions. + */ abstract void performRequestAsyncNoCatch(String method, String endpoint, Map params, HttpEntity entity, HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, ResponseListener responseListener, Header[] headers) throws IOException; diff --git a/client/rest/src/main/java/org/elasticsearch/client/Node.java b/client/rest/src/main/java/org/elasticsearch/client/Node.java index 7954b960835d3..abf75a1367290 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/Node.java +++ b/client/rest/src/main/java/org/elasticsearch/client/Node.java @@ -48,7 +48,9 @@ public class Node { private final Roles roles; /** - * Create a {@linkplain Node} with metadata. + * Create a {@linkplain Node} with metadata. All parameters except + * {@code host} are nullable and implementations of {@link NodeSelector} + * need to decide what to do in their absence. */ public Node(HttpHost host, List boundHosts, String version, Roles roles) { if (host == null) { @@ -67,6 +69,19 @@ public Node(HttpHost host) { this(host, null, null, null); } + /** + * Make a copy of this {@link Node} but replacing its {@link #getHost()} + * with the provided {@link HttpHost}. The provided host must be part of + * of {@link #getBoundHosts() bound hosts}. + */ + public Node withBoundHostAsHost(HttpHost boundHost) { + if (false == boundHosts.contains(boundHost)) { + throw new IllegalArgumentException(boundHost + " must be a bound host but wasn't in " + + boundHosts); + } + return new Node(boundHost, boundHosts, version, roles); + } + /** * Contact information for the host. */ @@ -75,7 +90,8 @@ public HttpHost getHost() { } /** - * Addresses that this host is bound to. + * Addresses that this host is bound to or {@code null} if we don't + * know which addresses are bound. */ public List getBoundHosts() { return boundHosts; @@ -97,19 +113,6 @@ public Roles getRoles() { return roles; } - /** - * Make a copy of this {@link Node} but replacing its {@link #getHost()} - * with the provided {@link HttpHost}. The provided host must be part of - * of {@link #getBoundHosts() bound hosts}. - */ - public Node withBoundHostAsHost(HttpHost boundHost) { - if (false == boundHosts.contains(boundHost)) { - throw new IllegalArgumentException(boundHost + " must be a bound host but wasn't in " - + boundHosts); - } - return new Node(boundHost, boundHosts, version, roles); - } - @Override public String toString() { StringBuilder b = new StringBuilder(); diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 4851580143e72..fa2caf2091cc2 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -157,7 +157,7 @@ private static Node[] hostsToNodes(HttpHost[] hosts) { * {@link #setHosts(HttpHost...)} if you have metadata about the hosts * like their Elasticsearch version of which roles they implement. */ - public void setNodes(Node... nodes) { + public synchronized void setNodes(Node... nodes) { if (nodes == null || nodes.length == 0) { throw new IllegalArgumentException("nodes must not be null or empty"); } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java index 550f833eca55f..c9ad4ac483cc0 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientActions.java @@ -181,12 +181,12 @@ void performRequestAsync(String method, String endpoint, Map par /** * Create a "stateless view" of this client that only runs requests on - * selected hosts. The result of this method is "stateless" it backs all - * of its requests to the {@link RestClient} that created it so there is - * no need to manage this view with try-with-resources and it does not - * extend {@link Closeable}. Closing the {@linkplain RestClient} that - * created the view disposes of the underlying connection, making the - * view useless. + * selected hosts. The result of this method is "stateless" because it + * backs all of its requests to the {@link RestClient} that created it + * so there is no need to manage this view with try-with-resources and + * it does not extend {@link Closeable}. Closing the + * {@linkplain RestClient} that created the view disposes of the + * underlying connection, making the view useless. */ RestClientActions withNodeSelector(NodeSelector nodeSelector); } diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java index fecc0d379aa2c..0f355d6f47876 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java @@ -24,7 +24,10 @@ import java.util.Arrays; +import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class NodeTests extends RestClientTestCase { @@ -57,5 +60,23 @@ public void testToString() { Arrays.asList(new HttpHost("1"), new HttpHost("2")), null, null).toString()); } - // TODO tests for equals and hashcode probably + public void testEqualsAndHashCode() { + HttpHost host = new HttpHost(randomAsciiAlphanumOfLength(5)); + Node node = new Node(host, + randomBoolean() ? null : singletonList(host), + randomBoolean() ? null : randomAsciiAlphanumOfLength(5), + randomBoolean() ? null : new Roles(true, true, true)); + assertFalse(node.equals(null)); + assertTrue(node.equals(node)); + assertEquals(node.hashCode(), node.hashCode()); + Node copy = new Node(host, node.getBoundHosts(), node.getVersion(), node.getRoles()); + assertTrue(node.equals(copy)); + assertEquals(node.hashCode(), copy.hashCode()); + assertFalse(node.equals(new Node(new HttpHost(host.toHostString() + "changed"), node.getBoundHosts(), + node.getVersion(), node.getRoles()))); + assertFalse(node.equals(new Node(host, Arrays.asList(host, new HttpHost(host.toHostString() + "changed")), + node.getVersion(), node.getRoles()))); + assertFalse(node.equals(new Node(host, node.getBoundHosts(), node.getVersion() + "changed", node.getRoles()))); + assertFalse(node.equals(new Node(host, node.getBoundHosts(), node.getVersion(), new Roles(false, false, false)))); + } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java index 3ef8b3274acd4..8eeb48eec95c2 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java @@ -60,8 +60,7 @@ public class RestClientMultipleHostsIntegTests extends RestClientTestCase { private static HttpHost[] httpHosts; private static String pathPrefixWithoutLeadingSlash; private static String pathPrefix; - - private RestClient restClient; + private static RestClient restClient; @BeforeClass public static void startHttpServer() throws Exception { @@ -80,11 +79,6 @@ public static void startHttpServer() throws Exception { httpServers[i] = httpServer; httpHosts[i] = new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort()); } - httpServers[0].createContext(pathPrefix + "/firstOnly", new ResponseHandler(200)); - } - - @Before - public void buildRestClient() throws Exception { RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); if (pathPrefix.length() > 0) { restClientBuilder.setPathPrefix((randomBoolean() ? "/" : "") + pathPrefixWithoutLeadingSlash); @@ -119,20 +113,18 @@ public void handle(HttpExchange httpExchange) throws IOException { } } - @After - public void stopRestClient() throws IOException { - restClient.close(); - } - @AfterClass public static void stopHttpServers() throws IOException { + restClient.close(); + restClient = null; for (HttpServer httpServer : httpServers) { httpServer.stop(0); } httpServers = null; } - private void stopRandomHost() { + @Before + public void stopRandomHost() { //verify that shutting down some hosts doesn't matter as long as one working host is left behind if (httpServers.length > 1 && randomBoolean()) { List updatedHttpServers = new ArrayList<>(httpServers.length - 1); @@ -150,7 +142,6 @@ private void stopRandomHost() { } public void testSyncRequests() throws IOException { - stopRandomHost(); int numRequests = randomIntBetween(5, 20); for (int i = 0; i < numRequests; i++) { final String method = RestClientTestUtil.randomHttpMethod(getRandom()); @@ -169,7 +160,6 @@ public void testSyncRequests() throws IOException { } public void testAsyncRequests() throws Exception { - stopRandomHost(); int numRequests = randomIntBetween(5, 20); final CountDownLatch latch = new CountDownLatch(numRequests); final List responses = new CopyOnWriteArrayList<>(); @@ -208,26 +198,37 @@ public void onFailure(Exception exception) { * test what happens after calling */ public void testWithNodeSelector() throws IOException { - /* - * note that this *doesn't* stopRandomHost(); because it might stop - * the first host which we use for testing the selector. - */ - NodeSelector firstPositionOnly = new NodeSelector() { - @Override - public boolean select(Node node) { - return httpHosts[0] == node.getHost(); - } - }; - RestClientActions withNodeSelector = restClient.withNodeSelector(firstPositionOnly); - Response response = withNodeSelector.performRequest("GET", "/firstOnly"); - assertEquals(httpHosts[0], response.getHost()); - restClient.close(); + RestClientActions withNodeSelector = restClient.withNodeSelector(FIRST_POSITION_NODE_SELECTOR); + int rounds = between(1, 10); + for (int i = 0; i < rounds; i++) { + /* + * Run the request more than once to verify that the + * NodeSelector overrides the round robin behavior. + */ + Response response = withNodeSelector.performRequest("GET", "/200"); + assertEquals(httpHosts[0], response.getHost()); + } + } + + /** + * Tests that stopping the {@link RestClient} backing the result of + * {@link RestClientActions#withNodeSelector(NodeSelector)} causes + * subsequent uses of the view to throw sensible exceptions. + */ + public void testStoppedView() throws IOException { + RestClientActions withNodeSelector; + try (RestClient toStop = RestClient.builder(httpHosts).build()) { + withNodeSelector = toStop.withNodeSelector(FIRST_POSITION_NODE_SELECTOR); + Response response = withNodeSelector.performRequest("GET", "/200"); + assertEquals(httpHosts[0], response.getHost()); + } try { - withNodeSelector.performRequest("GET", "/firstOnly"); + withNodeSelector.performRequest("GET", "/200"); fail("expected a failure"); } catch (IllegalStateException e) { assertThat(e.getMessage(), containsString("status: STOPPED")); } + } private static class TestResponse { @@ -254,4 +255,11 @@ Response getResponse() { throw new AssertionError("unexpected response " + response.getClass()); } } + + private static final NodeSelector FIRST_POSITION_NODE_SELECTOR = new NodeSelector() { + @Override + public boolean select(Node node) { + return httpHosts[0] == node.getHost(); + } + }; } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java index 97a171a3a4e00..3e2a8ee3e528b 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java @@ -43,10 +43,7 @@ import java.io.IOException; import java.net.SocketTimeoutException; -import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -59,7 +56,6 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -315,9 +311,15 @@ public boolean select(Node node) { } }; RestClientActions withNodeSelector = restClient.withNodeSelector(firstPositionOnly); - Response response = withNodeSelector.performRequest("GET", "/200"); - assertEquals(nodes[0].getHost(), response.getHost()); - restClient.close(); + int rounds = between(1, 10); + for (int i = 0; i < rounds; i++) { + /* + * Run the request more than once to verify that the + * NodeSelector overrides the round robin behavior. + */ + Response response = withNodeSelector.performRequest("GET", "/200"); + assertEquals(nodes[0].getHost(), response.getHost()); + } } public void testSetNodes() throws IOException { @@ -327,10 +329,16 @@ public void testSetNodes() throws IOException { newNodes[i] = new Node(nodes[i].getHost(), null, null, roles); } restClient.setNodes(newNodes); - Response response = restClient - .withNodeSelector(NodeSelector.NOT_MASTER_ONLY) - .performRequest("GET", "/200"); - assertEquals(newNodes[0].getHost(), response.getHost()); + RestClientActions withNodeSelector = restClient.withNodeSelector(NodeSelector.NOT_MASTER_ONLY); + int rounds = between(1, 10); + for (int i = 0; i < rounds; i++) { + /* + * Run the request more than once to verify that the + * NodeSelector overrides the round robin behavior. + */ + Response response = withNodeSelector.performRequest("GET", "/200"); + assertEquals(newNodes[0].getHost(), response.getHost()); + } } private static String randomErrorRetryEndpoint() { From a5a47a202ce1f945b2d27f256f1cd28e89e478e3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 27 Mar 2018 10:27:53 -0400 Subject: [PATCH 18/27] Explain and rework a bit --- .../java/org/elasticsearch/client/Node.java | 41 ++++++++++++------- .../org/elasticsearch/client/NodeTests.java | 21 ++++------ .../sniff/ElasticsearchHostsSniffer.java | 6 +++ .../test/rest/ESRestTestCase.java | 9 +++- .../test/rest/ESRestTestCaseTests.java | 2 +- 5 files changed, 51 insertions(+), 28 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/Node.java b/client/rest/src/main/java/org/elasticsearch/client/Node.java index abf75a1367290..57362f88c41de 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/Node.java +++ b/client/rest/src/main/java/org/elasticsearch/client/Node.java @@ -19,6 +19,9 @@ package org.elasticsearch.client; +import static java.util.Collections.unmodifiableList; + +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -33,7 +36,9 @@ public class Node { */ private final HttpHost host; /** - * Addresses that this host is bound to. + * Addresses on which the host is listening. These are useful to have + * around because they allow you to find a host based on any address it + * is listening on. */ private final List boundHosts; /** @@ -70,16 +75,23 @@ public Node(HttpHost host) { } /** - * Make a copy of this {@link Node} but replacing its {@link #getHost()} - * with the provided {@link HttpHost}. The provided host must be part of - * of {@link #getBoundHosts() bound hosts}. + * Make a copy of this {@link Node} but replacing its + * {@link #getHost() host}. Use this when the sniffing implementation + * returns returns a {@link #getHost() host} that is not useful to the + * client. */ - public Node withBoundHostAsHost(HttpHost boundHost) { - if (false == boundHosts.contains(boundHost)) { - throw new IllegalArgumentException(boundHost + " must be a bound host but wasn't in " - + boundHosts); + public Node withHost(HttpHost host) { + /* + * If the new host isn't in the bound hosts list we add it so the + * result looks sane. + */ + List boundHosts = this.boundHosts; + if (false == boundHosts.contains(host)) { + boundHosts = new ArrayList<>(boundHosts); + boundHosts.add(host); + boundHosts = unmodifiableList(boundHosts); } - return new Node(boundHost, boundHosts, version, roles); + return new Node(host, boundHosts, version, roles); } /** @@ -90,8 +102,9 @@ public HttpHost getHost() { } /** - * Addresses that this host is bound to or {@code null} if we don't - * know which addresses are bound. + * Addresses on which the host is listening. These are useful to have + * around because they allow you to find a host based on any address it + * is listening on. */ public List getBoundHosts() { return boundHosts; @@ -136,14 +149,14 @@ public boolean equals(Object obj) { } Node other = (Node) obj; return host.equals(other.host) + && Objects.equals(boundHosts, other.boundHosts) && Objects.equals(version, other.version) - && Objects.equals(roles, other.roles) - && Objects.equals(boundHosts, other.boundHosts); + && Objects.equals(roles, other.roles); } @Override public int hashCode() { - return Objects.hash(host, version, roles, boundHosts); + return Objects.hash(host, boundHosts, version, roles); } /** diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java index 0f355d6f47876..c328ef7fa5584 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java @@ -28,26 +28,23 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; public class NodeTests extends RestClientTestCase { - public void testWithBoundHostAsHost() { + public void testWithHost() { HttpHost h1 = new HttpHost("1"); HttpHost h2 = new HttpHost("2"); HttpHost h3 = new HttpHost("3"); - Node n = new Node(h1, Arrays.asList(h1, h2, h3), randomAsciiAlphanumOfLength(5), + Node n = new Node(h1, Arrays.asList(h1, h2), randomAsciiAlphanumOfLength(5), new Roles(randomBoolean(), randomBoolean(), randomBoolean())); - assertEquals(h2, n.withBoundHostAsHost(h2).getHost()); - assertEquals(n.getBoundHosts(), n.withBoundHostAsHost(h2).getBoundHosts()); - try { - n.withBoundHostAsHost(new HttpHost("4")); - fail("expected failure"); - } catch (IllegalArgumentException e) { - assertEquals("http://4 must be a bound host but wasn't in [http://1, http://2, http://3]", - e.getMessage()); - } + // Host is in nthe bound hosts list + assertEquals(h2, n.withHost(h2).getHost()); + assertEquals(n.getBoundHosts(), n.withHost(h2).getBoundHosts()); + + // Host not in the bound hosts list + assertEquals(h3, n.withHost(h3).getHost()); + assertEquals(Arrays.asList(h1, h2, h3), n.withHost(h3).getBoundHosts()); } public void testToString() { diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java index 7a6e4d3493977..1de6fb468a9e6 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java @@ -125,6 +125,12 @@ private List readHosts(HttpEntity entity) throws IOException { private static void readHost(String nodeId, JsonParser parser, Scheme scheme, List nodes) throws IOException { HttpHost publishedHost = null; + /* + * We sniff the bound hosts so we can look up the node based on any + * address on which it is listening. This is useful in Elasticsearch's + * test framework where we sometimes publish ipv6 addresses but the + * tests contact the node on ipv4. + */ List boundHosts = new ArrayList<>(); String fieldName = null; String version = null; diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index b92dcf1ba3d56..a82fa177c47ce 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -580,12 +580,19 @@ static void attachSniffedMetadataOnClient(RestClient client, Node[] originalNode .collect(Collectors.toSet()); List sniffed = new ArrayList<>(); for (Node node : nodesWithMetadata) { + /* + * getHost is the publish_address of the node which, sometimes, is + * ipv6 and, sometimes, our original address for the node is ipv4. + * In that case the ipv4 address should be in getBoundHosts. If it + * isn't then we'll end up without the right number of hosts which + * will fail down below with a pretty error message. + */ if (originalHosts.contains(node.getHost())) { sniffed.add(node); } else { for (HttpHost bound : node.getBoundHosts()) { if (originalHosts.contains(bound)) { - sniffed.add(node.withBoundHostAsHost(bound)); + sniffed.add(node.withHost(bound)); break; } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java index 2bc6153b34878..5f0ced12f418b 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java @@ -58,7 +58,7 @@ public void testAttachSniffedMetadataOnClientOk() { verify(client).setNodes(new Node[] { nodesWithMetadata.get(0), nodesWithMetadata.get(1), - nodesWithMetadata.get(2).withBoundHostAsHost(new HttpHost("3")), + nodesWithMetadata.get(2).withHost(new HttpHost("3")), }); } From a27317bf4fa897b46e57d3b650446496ac8a9a25 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 27 Mar 2018 10:37:37 -0400 Subject: [PATCH 19/27] Move --- .../test/rest/ESRestTestCase.java | 76 ---------------- .../rest/yaml/ESClientYamlSuiteTestCase.java | 76 ++++++++++++++++ .../test/rest/ESRestTestCaseTests.java | 86 ------------------- .../yaml/ESClientYamlSuiteTestCaseTests.java | 60 +++++++++++++ 4 files changed, 136 insertions(+), 162 deletions(-) delete mode 100644 test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index a82fa177c47ce..befc21eb1f697 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -32,13 +32,10 @@ import org.apache.http.ssl.SSLContexts; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksAction; -import org.elasticsearch.client.Node; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; -import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer; -import org.elasticsearch.client.sniff.Sniffer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.settings.Settings; @@ -67,16 +64,13 @@ import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonMap; @@ -538,74 +532,4 @@ protected static Map getAsMap(final String endpoint) throws IOEx assertNotNull(responseEntity); return responseEntity; } - - /** - * Sniff the cluster for host metadata if it hasn't already been sniffed. This isn't the - * same thing as using the {@link Sniffer} because: - *
    - *
  • It doesn't replace the hosts that that {@link #client} communicates with - *
  • If there is already host metadata it skips running. This behavior isn't - * thread safe but it doesn't have to be for our tests. - *
- */ - protected void sniffHostMetadata(RestClient client) throws IOException { - Node[] nodes = client.getNodes(); - boolean allHaveRoles = true; - for (Node node : nodes) { - if (node.getRoles() == null) { - allHaveRoles = false; - break; - } - } - if (allHaveRoles) { - // We already have resolved metadata. - return; - } - // No resolver, sniff one time and resolve metadata against the results - ElasticsearchHostsSniffer.Scheme scheme = - ElasticsearchHostsSniffer.Scheme.valueOf(getProtocol().toUpperCase(Locale.ROOT)); - /* - * We don't want to change the list of nodes that the client communicates with - * because that'd just be rude. So instead we replace the nodes with nodes the - * that - */ - ElasticsearchHostsSniffer sniffer = new ElasticsearchHostsSniffer( - adminClient, ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT, scheme); - attachSniffedMetadataOnClient(client, nodes, sniffer.sniffHosts()); - } - - static void attachSniffedMetadataOnClient(RestClient client, Node[] originalNodes, List nodesWithMetadata) { - Set originalHosts = Arrays.stream(originalNodes) - .map(Node::getHost) - .collect(Collectors.toSet()); - List sniffed = new ArrayList<>(); - for (Node node : nodesWithMetadata) { - /* - * getHost is the publish_address of the node which, sometimes, is - * ipv6 and, sometimes, our original address for the node is ipv4. - * In that case the ipv4 address should be in getBoundHosts. If it - * isn't then we'll end up without the right number of hosts which - * will fail down below with a pretty error message. - */ - if (originalHosts.contains(node.getHost())) { - sniffed.add(node); - } else { - for (HttpHost bound : node.getBoundHosts()) { - if (originalHosts.contains(bound)) { - sniffed.add(node.withHost(bound)); - break; - } - } - } - } - int missing = originalNodes.length - sniffed.size(); - if (missing > 0) { - List hosts = Arrays.stream(originalNodes) - .map(Node::getHost) - .collect(Collectors.toList()); - throw new IllegalStateException("Didn't sniff metadata for all nodes. Wanted metadata for " - + hosts + " but got " + sniffed); - } - client.setNodes(sniffed.toArray(new Node[0])); - } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 49b3ab4fdaf27..a72d25cdf0545 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -22,9 +22,11 @@ import com.carrotsearch.randomizedtesting.RandomizedTest; import org.apache.http.HttpHost; import org.elasticsearch.Version; +import org.elasticsearch.client.Node; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.io.PathUtils; @@ -42,12 +44,15 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * Runs a suite of yaml tests shared with all the official Elasticsearch clients against against an elasticsearch cluster. @@ -356,4 +361,75 @@ private String errorMessage(ExecutableSection executableSection, Throwable t) { protected boolean randomizeContentType() { return true; } + + + /** + * Sniff the cluster for host metadata if it hasn't already been sniffed. This isn't the + * same thing as using the {@link Sniffer} because: + *
    + *
  • It doesn't replace the hosts that that {@link #client} communicates with + *
  • If there is already host metadata it skips running. This behavior isn't + * thread safe but it doesn't have to be for our tests. + *
+ */ + private void sniffHostMetadata(RestClient client) throws IOException { + Node[] nodes = client.getNodes(); + boolean allHaveRoles = true; + for (Node node : nodes) { + if (node.getRoles() == null) { + allHaveRoles = false; + break; + } + } + if (allHaveRoles) { + // We already have resolved metadata. + return; + } + // No resolver, sniff one time and resolve metadata against the results + ElasticsearchHostsSniffer.Scheme scheme = + ElasticsearchHostsSniffer.Scheme.valueOf(getProtocol().toUpperCase(Locale.ROOT)); + /* + * We don't want to change the list of nodes that the client communicates with + * because that'd just be rude. So instead we replace the nodes with nodes the + * that + */ + ElasticsearchHostsSniffer sniffer = new ElasticsearchHostsSniffer( + adminClient(), ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT, scheme); + attachSniffedMetadataOnClient(client, nodes, sniffer.sniffHosts()); + } + + static void attachSniffedMetadataOnClient(RestClient client, Node[] originalNodes, List nodesWithMetadata) { + Set originalHosts = Arrays.stream(originalNodes) + .map(Node::getHost) + .collect(Collectors.toSet()); + List sniffed = new ArrayList<>(); + for (Node node : nodesWithMetadata) { + /* + * getHost is the publish_address of the node which, sometimes, is + * ipv6 and, sometimes, our original address for the node is ipv4. + * In that case the ipv4 address should be in getBoundHosts. If it + * isn't then we'll end up without the right number of hosts which + * will fail down below with a pretty error message. + */ + if (originalHosts.contains(node.getHost())) { + sniffed.add(node); + } else { + for (HttpHost bound : node.getBoundHosts()) { + if (originalHosts.contains(bound)) { + sniffed.add(node.withHost(bound)); + break; + } + } + } + } + int missing = originalNodes.length - sniffed.size(); + if (missing > 0) { + List hosts = Arrays.stream(originalNodes) + .map(Node::getHost) + .collect(Collectors.toList()); + throw new IllegalStateException("Didn't sniff metadata for all nodes. Wanted metadata for " + + hosts + " but got " + sniffed); + } + client.setNodes(sniffed.toArray(new Node[0])); + } } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java deleted file mode 100644 index 5f0ced12f418b..0000000000000 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/ESRestTestCaseTests.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test.rest; - -import org.apache.http.HttpHost; -import org.elasticsearch.client.Node; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.test.ESTestCase; - -import java.util.Arrays; -import java.util.List; - -import static java.util.Collections.emptyList; -import static org.mockito.Mockito.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; - -public class ESRestTestCaseTests extends ESTestCase { - public void testAttachSniffedMetadataOnClientOk() { - RestClient client = mock(RestClient.class); - Node[] originalNodes = new Node[] { - new Node(new HttpHost("1")), - new Node(new HttpHost("2")), - new Node(new HttpHost("3")), - }; - List nodesWithMetadata = Arrays.asList(new Node[] { - // This node matches exactly: - new Node(new HttpHost("1"), emptyList(), randomAlphaOfLength(5), randomRoles()), - // This node also matches exactly but has bound hosts which don't matter: - new Node(new HttpHost("2"), Arrays.asList(new HttpHost("2"), new HttpHost("not2")), - randomAlphaOfLength(5), randomRoles()), - // This node's host doesn't match but one of its published hosts does so - // we return a modified version of it: - new Node(new HttpHost("not3"), Arrays.asList(new HttpHost("not3"), new HttpHost("3")), - randomAlphaOfLength(5), randomRoles()), - // This node isn't in the original list so it isn't added: - new Node(new HttpHost("4"), emptyList(), randomAlphaOfLength(5), randomRoles()), - }); - ESRestTestCase.attachSniffedMetadataOnClient(client, originalNodes, nodesWithMetadata); - verify(client).setNodes(new Node[] { - nodesWithMetadata.get(0), - nodesWithMetadata.get(1), - nodesWithMetadata.get(2).withHost(new HttpHost("3")), - }); - } - - public void testAttachSniffedMetadataOnClientNotEnoughNodes() { - // Try a version of the call that should fail because it doesn't have all the results - RestClient client = mock(RestClient.class); - Node[] originalNodes = new Node[] { - new Node(new HttpHost("1")), - new Node(new HttpHost("2")), - }; - List nodesWithMetadata = Arrays.asList(new Node[] { - // This node matches exactly: - new Node(new HttpHost("1"), emptyList(), "v", new Node.Roles(true, true, true)), - }); - IllegalStateException e = expectThrows(IllegalStateException.class, () -> - ESRestTestCase.attachSniffedMetadataOnClient(client, originalNodes, nodesWithMetadata)); - assertEquals(e.getMessage(), "Didn't sniff metadata for all nodes. Wanted metadata for " - + "[http://1, http://2] but got [[host=http://1, bound=[], version=v, roles=mdi]]"); - verify(client, never()).setNodes(any(Node[].class)); - } - - private Node.Roles randomRoles() { - return new Node.Roles(randomBoolean(), randomBoolean(), randomBoolean()); - } -} diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java index ae64dbc893d81..825f05f78b0b6 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java @@ -20,14 +20,24 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.http.HttpHost; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.RestClient; import org.elasticsearch.test.ESTestCase; +import static java.util.Collections.emptyList; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.greaterThan; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; public class ESClientYamlSuiteTestCaseTests extends ESTestCase { @@ -91,4 +101,54 @@ private static void assertSingleFile(Set files, String dirName, String fil assertThat(file.getFileName().toString(), equalTo(fileName)); assertThat(file.toAbsolutePath().getParent().getFileName().toString(), equalTo(dirName)); } + + public void testAttachSniffedMetadataOnClientOk() { + RestClient client = mock(RestClient.class); + Node[] originalNodes = new Node[] { + new Node(new HttpHost("1")), + new Node(new HttpHost("2")), + new Node(new HttpHost("3")), + }; + List nodesWithMetadata = Arrays.asList(new Node[] { + // This node matches exactly: + new Node(new HttpHost("1"), emptyList(), randomAlphaOfLength(5), randomRoles()), + // This node also matches exactly but has bound hosts which don't matter: + new Node(new HttpHost("2"), Arrays.asList(new HttpHost("2"), new HttpHost("not2")), + randomAlphaOfLength(5), randomRoles()), + // This node's host doesn't match but one of its published hosts does so + // we return a modified version of it: + new Node(new HttpHost("not3"), Arrays.asList(new HttpHost("not3"), new HttpHost("3")), + randomAlphaOfLength(5), randomRoles()), + // This node isn't in the original list so it isn't added: + new Node(new HttpHost("4"), emptyList(), randomAlphaOfLength(5), randomRoles()), + }); + ESClientYamlSuiteTestCase.attachSniffedMetadataOnClient(client, originalNodes, nodesWithMetadata); + verify(client).setNodes(new Node[] { + nodesWithMetadata.get(0), + nodesWithMetadata.get(1), + nodesWithMetadata.get(2).withHost(new HttpHost("3")), + }); + } + + public void testAttachSniffedMetadataOnClientNotEnoughNodes() { + // Try a version of the call that should fail because it doesn't have all the results + RestClient client = mock(RestClient.class); + Node[] originalNodes = new Node[] { + new Node(new HttpHost("1")), + new Node(new HttpHost("2")), + }; + List nodesWithMetadata = Arrays.asList(new Node[] { + // This node matches exactly: + new Node(new HttpHost("1"), emptyList(), "v", new Node.Roles(true, true, true)), + }); + IllegalStateException e = expectThrows(IllegalStateException.class, () -> + ESClientYamlSuiteTestCase.attachSniffedMetadataOnClient(client, originalNodes, nodesWithMetadata)); + assertEquals(e.getMessage(), "Didn't sniff metadata for all nodes. Wanted metadata for " + + "[http://1, http://2] but got [[host=http://1, bound=[], version=v, roles=mdi]]"); + verify(client, never()).setNodes(any(Node[].class)); + } + + private Node.Roles randomRoles() { + return new Node.Roles(randomBoolean(), randomBoolean(), randomBoolean()); + } } From e931d4c7cb449959a1c17831a1c1ae91f6d2dac9 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 27 Mar 2018 11:22:59 -0400 Subject: [PATCH 20/27] Fixup test --- .../RestClientMultipleHostsIntegTests.java | 53 +++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java index 8eeb48eec95c2..03d308c3dc3a7 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java @@ -25,12 +25,12 @@ import org.apache.http.HttpHost; import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement; import org.elasticsearch.mocksocket.MockHttpServer; -import org.junit.After; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; import java.io.IOException; +import java.net.ConnectException; import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -58,6 +58,7 @@ public class RestClientMultipleHostsIntegTests extends RestClientTestCase { private static HttpServer[] httpServers; private static HttpHost[] httpHosts; + private static boolean stoppedFirstHost = false; private static String pathPrefixWithoutLeadingSlash; private static String pathPrefix; private static RestClient restClient; @@ -79,11 +80,15 @@ public static void startHttpServer() throws Exception { httpServers[i] = httpServer; httpHosts[i] = new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort()); } + restClient = buildRestClient(); + } + + private static RestClient buildRestClient() { RestClientBuilder restClientBuilder = RestClient.builder(httpHosts); if (pathPrefix.length() > 0) { restClientBuilder.setPathPrefix((randomBoolean() ? "/" : "") + pathPrefixWithoutLeadingSlash); } - restClient = restClientBuilder.build(); + return restClientBuilder.build(); } private static HttpServer createHttpServer() throws Exception { @@ -129,6 +134,9 @@ public void stopRandomHost() { if (httpServers.length > 1 && randomBoolean()) { List updatedHttpServers = new ArrayList<>(httpServers.length - 1); int nodeIndex = randomInt(httpServers.length - 1); + if (0 == nodeIndex) { + stoppedFirstHost = true; + } for (int i = 0; i < httpServers.length; i++) { HttpServer httpServer = httpServers[i]; if (i == nodeIndex) { @@ -198,15 +206,14 @@ public void onFailure(Exception exception) { * test what happens after calling */ public void testWithNodeSelector() throws IOException { - RestClientActions withNodeSelector = restClient.withNodeSelector(FIRST_POSITION_NODE_SELECTOR); + RestClientActions withNodeSelector = restClient.withNodeSelector(firstPositionNodeSelector()); int rounds = between(1, 10); for (int i = 0; i < rounds; i++) { /* * Run the request more than once to verify that the * NodeSelector overrides the round robin behavior. */ - Response response = withNodeSelector.performRequest("GET", "/200"); - assertEquals(httpHosts[0], response.getHost()); + performRequestAndAssertOnFirstHost(withNodeSelector); } } @@ -217,10 +224,10 @@ public void testWithNodeSelector() throws IOException { */ public void testStoppedView() throws IOException { RestClientActions withNodeSelector; - try (RestClient toStop = RestClient.builder(httpHosts).build()) { - withNodeSelector = toStop.withNodeSelector(FIRST_POSITION_NODE_SELECTOR); - Response response = withNodeSelector.performRequest("GET", "/200"); - assertEquals(httpHosts[0], response.getHost()); + // Build our own RestClient for this test because we're going to close it. + try (RestClient toStop = buildRestClient()) { + withNodeSelector = toStop.withNodeSelector(firstPositionNodeSelector()); + performRequestAndAssertOnFirstHost(withNodeSelector); } try { withNodeSelector.performRequest("GET", "/200"); @@ -228,6 +235,20 @@ public void testStoppedView() throws IOException { } catch (IllegalStateException e) { assertThat(e.getMessage(), containsString("status: STOPPED")); } + } + + private void performRequestAndAssertOnFirstHost(RestClientActions withNodeSelector) throws IOException { + if (stoppedFirstHost) { + try { + withNodeSelector.performRequest("GET", "/200"); + fail("expected to fail to connect"); + } catch (ConnectException e) { + assertEquals("Connection refused", e.getMessage()); + } + } else { + Response response = withNodeSelector.performRequest("GET", "/200"); + assertEquals(httpHosts[0], response.getHost()); + } } @@ -256,10 +277,12 @@ Response getResponse() { } } - private static final NodeSelector FIRST_POSITION_NODE_SELECTOR = new NodeSelector() { - @Override - public boolean select(Node node) { - return httpHosts[0] == node.getHost(); - } - }; + private NodeSelector firstPositionNodeSelector() { + return new NodeSelector() { + @Override + public boolean select(Node node) { + return httpHosts[0] == node.getHost(); + } + }; + } } From 398f9a339f24b997dd493c51f9830ec9a1fa3690 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 27 Mar 2018 15:13:04 -0400 Subject: [PATCH 21/27] Rework NodeSelector so it takes a list --- .../elasticsearch/client/NodeSelector.java | 47 +++-- .../org/elasticsearch/client/RestClient.java | 189 ++++++++---------- .../elasticsearch/client/RestClientView.java | 2 +- .../client/NodeSelectorTests.java | 24 ++- .../RestClientMultipleHostsIntegTests.java | 11 +- .../client/RestClientMultipleHostsTests.java | 9 +- .../elasticsearch/client/RestClientTests.java | 142 +++++++------ .../test/rest/yaml/section/DoSection.java | 20 +- .../rest/yaml/section/DoSectionTests.java | 13 +- 9 files changed, 242 insertions(+), 215 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java index ac97071023b02..cea3886a32333 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java +++ b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java @@ -19,6 +19,8 @@ package org.elasticsearch.client; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; /** @@ -27,18 +29,23 @@ */ public interface NodeSelector { /** - * Return {@code true} if the provided node should be used for requests, - * {@code false} otherwise. + * Select the {@link Node}s to which to send requests. + * + * @param nodes an unmodifiable list of {@linkplain Node}s in the order + * that the {@link RestClient} would prefer to use them + * @return a subset of the provided list of {@linkplain Node}s that the + * selector approves of, in the order that the selector would prefer + * to use them. */ - boolean select(Node node); + List select(List nodes); /** * Selector that matches any node. */ NodeSelector ANY = new NodeSelector() { @Override - public boolean select(Node node) { - return true; + public List select(List nodes) { + return nodes; } @Override @@ -50,13 +57,19 @@ public String toString() { /** * Selector that matches any node that has metadata and doesn't * have the {@code master} role OR it has the data {@code data} - * role. + * role. It does not reorder the nodes sent to it. */ NodeSelector NOT_MASTER_ONLY = new NodeSelector() { @Override - public boolean select(Node node) { - return node.getRoles() != null - && (false == node.getRoles().hasMasterEligible() || node.getRoles().hasData()); + public List select(List nodes) { + List subset = new ArrayList<>(nodes.size()); + for (Node node : nodes) { + if (node.getRoles() == null) continue; + if (false == node.getRoles().hasMasterEligible() || node.getRoles().hasData()) { + subset.add(node); + } + } + return subset; } @Override @@ -66,26 +79,28 @@ public String toString() { }; /** - * Selector that returns {@code true} of both of its provided - * selectors return {@code true}, otherwise {@code false}. + * Selector that composes two selectors, running the "right" most selector + * first and then running the "left" selector on the results of the "right" + * selector. */ - class And implements NodeSelector { + class Compose implements NodeSelector { private final NodeSelector lhs; private final NodeSelector rhs; - public And(NodeSelector lhs, NodeSelector rhs) { + public Compose(NodeSelector lhs, NodeSelector rhs) { this.lhs = Objects.requireNonNull(lhs, "lhs is required"); this.rhs = Objects.requireNonNull(rhs, "rhs is required"); } @Override - public boolean select(Node node) { - return lhs.select(node) && rhs.select(node); + public List select(List nodes) { + return lhs.select(rhs.select(nodes)); } @Override public String toString() { - return lhs + " AND " + rhs; + // . as in haskell's "compose" operator + return lhs + "." + rhs; } } } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index fa2caf2091cc2..76a3e9ec167ed 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -45,6 +45,8 @@ import org.apache.http.nio.protocol.HttpAsyncRequestProducer; import org.apache.http.nio.protocol.HttpAsyncResponseConsumer; +import static java.util.Collections.singletonList; + import java.io.Closeable; import java.io.IOException; import java.net.URI; @@ -103,7 +105,7 @@ public class RestClient extends AbstractRestClientActions implements Closeable { private final AtomicInteger lastNodeIndex = new AtomicInteger(0); private final ConcurrentMap blacklist = new ConcurrentHashMap<>(); private final FailureListener failureListener; - private volatile NodeTuple> nodeTuple; + private volatile NodeTuple> nodeTuple; RestClient(CloseableHttpAsyncClient client, long maxRetryTimeoutMillis, Header[] defaultHeaders, Node[] nodes, String pathPrefix, FailureListener failureListener) { @@ -161,17 +163,16 @@ public synchronized void setNodes(Node... nodes) { if (nodes == null || nodes.length == 0) { throw new IllegalArgumentException("nodes must not be null or empty"); } - Set newNodes = new HashSet<>(); AuthCache authCache = new BasicAuthCache(); for (Node node : nodes) { if (node == null) { throw new IllegalArgumentException("node cannot be null"); } - newNodes.add(node); authCache.put(node.getHost(), new BasicScheme()); } - this.nodeTuple = new NodeTuple<>(Collections.unmodifiableSet(newNodes), authCache); + this.nodeTuple = new NodeTuple<>(Collections.unmodifiableList( + Arrays.asList(nodes)), authCache); this.blacklist.clear(); } @@ -360,128 +361,98 @@ private void setHeaders(HttpRequest httpRequest, Header[] requestHeaders) { * @throws IOException if no nodes are available */ private NodeTuple> nextNode(NodeSelector nodeSelector) throws IOException { - int attempts = 0; - NextHostsResult result; - /* - * Try to fetch the hosts to which we can send the request. It is possible that - * this returns an empty collection because of concurrent modification to the - * blacklist. - */ - do { - final NodeTuple> nodeTuple = this.nodeTuple; - result = nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, System.nanoTime(), nodeSelector); - if (result.hosts == null) { - if (logger.isDebugEnabled()) { - logger.debug("No nodes avialable. Will retry. Failure is " + result.describeFailure()); - } - } else { - // Success! - return new NodeTuple<>(result.hosts.iterator(), nodeTuple.authCache); - } - attempts++; - } while (attempts < MAX_NEXT_NODES_ATTEMPTS); - throw new IOException("No nodes available for request. Last failure was " + result.describeFailure()); + NodeTuple> nodeTuple = this.nodeTuple; + List hosts = selectHosts(nodeTuple, blacklist, lastNodeIndex, System.nanoTime(), nodeSelector); + return new NodeTuple<>(hosts.iterator(), nodeTuple.authCache); } - static class NextHostsResult { - /** - * Number of nodes filtered from the list because they are - * dead. - */ - int blacklisted = 0; - /** - * Number of nodes filtered from the list because the. - * {@link NodeSelector} didn't approve of them. - */ - int selectorRejected = 0; - /** - * Number of nodes that could not be revived because the - * {@link NodeSelector} didn't approve of them. - */ - int selectorBlockedRevival = 0; - /** - * {@code null} if we failed to find any nodes, a list of - * nodes to use if we found any. - */ - Collection hosts = null; + static List selectHosts(NodeTuple> nodeTuple, + Map blacklist, AtomicInteger lastNodeIndex, + long now, NodeSelector nodeSelector) throws IOException { + class DeadNodeAndRevival { + final Node node; + final long nanosUntilRevival; + + DeadNodeAndRevival(Node node, long nanosUntilRevival) { + this.node = node; + this.nanosUntilRevival = nanosUntilRevival; + } - public String describeFailure() { - assert hosts == null : "describeFailure shouldn't be called with successful request"; - return "[blacklisted=" + blacklisted - + ", selectorRejected=" + selectorRejected - + ", selectorBlockedRevival=" + selectorBlockedRevival + "]]"; + @Override + public String toString() { + return node.toString(); + } } - } - static NextHostsResult nextHostsOneTime(NodeTuple> nodeTuple, - Map blacklist, AtomicInteger lastNodesIndex, - long now, NodeSelector nodeSelector) { - NextHostsResult result = new NextHostsResult(); - // TODO there has to be a better way! - Map hostToNode = new HashMap<>(nodeTuple.nodes.size()); + + /* + * Sort the nodes into living and dead lists. + */ + List livingNodes = new ArrayList<>(nodeTuple.nodes.size() - blacklist.size()); + List deadNodes = new ArrayList<>(blacklist.size()); for (Node node : nodeTuple.nodes) { - hostToNode.put(node.getHost(), node); - } - Set filteredNodes = new HashSet<>(nodeTuple.nodes); - for (Map.Entry entry : blacklist.entrySet()) { - if (now - entry.getValue().getDeadUntilNanos() < 0) { - filteredNodes.remove(hostToNode.get(entry.getKey())); - result.blacklisted++; + DeadHostState deadness = blacklist.get(node.getHost()); + if (deadness == null) { + livingNodes.add(node); + continue; } - } - for (Iterator nodeItr = filteredNodes.iterator(); nodeItr.hasNext();) { - final Node node = nodeItr.next(); - if (false == nodeSelector.select(node)) { - nodeItr.remove(); - result.selectorRejected++; + long nanosUntilRevival = now - deadness.getDeadUntilNanos(); + if (nanosUntilRevival > 0) { + livingNodes.add(node); + continue; } + deadNodes.add(new DeadNodeAndRevival(node, nanosUntilRevival)); } - if (false == filteredNodes.isEmpty()) { + + if (false == livingNodes.isEmpty()) { /* - * Normal case: we have at least one non-dead node that the nodeSelector - * is fine with. Rotate the list so repeated requests with the same blacklist - * and the same selector round robin. If you use a different NodeSelector - * or a node goes dark then the round robin won't be perfect but that should - * be fine. + * Normal state: there is at least one living node. Rotate the + * list so subsequent requests to will see prefer the nodes in + * a different order then run them through the NodeSelector so + * it can have its say in which nodes are ok and their ordering. + * If the selector is ok with any over the living nodes then use + * them for the request. */ - List rotatedHosts = new ArrayList<>(filteredNodes.size()); - for (Node node : filteredNodes) { - rotatedHosts.add(node.getHost()); + Collections.rotate(livingNodes, lastNodeIndex.getAndIncrement()); + List selectedLivingNodes = nodeSelector.select(livingNodes); + if (false == selectedLivingNodes.isEmpty()) { + List hosts = new ArrayList<>(selectedLivingNodes.size()); + for (Node node : selectedLivingNodes) { + hosts.add(node.getHost()); + } + return hosts; } - int i = lastNodesIndex.getAndIncrement(); - Collections.rotate(rotatedHosts, i); - result.hosts = rotatedHosts; - return result; } + /* - * Last resort: If there are no good nodes to use, return a single dead one, - * the one that's closest to being retried *and* matches the selector. + * Last resort: If there are no good nodes to use, either because + * the selector rejected all the living nodes or because there aren't + * any living ones. Either way, we want to revive a single dead node + * that the NodeSelectors are OK with. We do this by sorting the dead + * nodes by their revival time and passing them through the + * NodeSelector so it can have its say in which nodes are ok and their + * ordering. If the selector is ok with any of the nodes then use just + * the first one in the list because we only want to revive a single + * node. */ - List> sortedHosts = new ArrayList<>(blacklist.entrySet()); - if (sortedHosts.isEmpty()) { - // There are no dead hosts to revive. Return a failed result and we'll retry. - return result; - } - Collections.sort(sortedHosts, new Comparator>() { - @Override - public int compare(Map.Entry o1, Map.Entry o2) { - return Long.compare(o1.getValue().getDeadUntilNanos(), o2.getValue().getDeadUntilNanos()); - } - }); - Iterator> nodeItr = sortedHosts.iterator(); - while (nodeItr.hasNext()) { - final HttpHost deadHost = nodeItr.next().getKey(); - Node node = hostToNode.get(deadHost); - if (node != null && nodeSelector.select(node)) { - if (logger.isTraceEnabled()) { - logger.trace("resurrecting host [" + deadHost + "]"); + if (false == deadNodes.isEmpty()) { + Collections.sort(deadNodes, new Comparator() { + @Override + public int compare(DeadNodeAndRevival lhs, DeadNodeAndRevival rhs) { + return Long.compare(rhs.nanosUntilRevival, lhs.nanosUntilRevival); } - result.hosts = Collections.singleton(deadHost); - return result; - } else { - result.selectorBlockedRevival++; + }); + + List selectedDeadNodes = new ArrayList<>(deadNodes.size()); + for (DeadNodeAndRevival n : deadNodes) { + selectedDeadNodes.add(n.node); + } + selectedDeadNodes = nodeSelector.select(selectedDeadNodes); + if (false == selectedDeadNodes.isEmpty()) { + return singletonList(selectedDeadNodes.get(0).getHost()); } } - return result; + throw new IOException("NodeSelector [" + nodeSelector + "] rejcted all nodes, " + + "living " + livingNodes + " and dead " + deadNodes); } /** diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java index 0075241a9e690..87b78024646e0 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java @@ -51,7 +51,7 @@ final SyncResponseListener syncResponseListener() { @Override public final RestClientView withNodeSelector(final NodeSelector newNodeSelector) { - return new RestClientView(delegate, new NodeSelector.And(nodeSelector, newNodeSelector)); + return new RestClientView(delegate, new NodeSelector.Compose(nodeSelector, newNodeSelector)); } @Override diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java index d7778a482cd0e..dfbcd9461c580 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java @@ -22,14 +22,21 @@ import org.apache.http.HttpHost; import org.elasticsearch.client.Node.Roles; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; + +import static org.junit.Assert.assertEquals; public class NodeSelectorTests extends RestClientTestCase { public void testAny() { - assertTrue(NodeSelector.ANY.select(dummyNode(randomBoolean(), randomBoolean(), randomBoolean()))); + List nodes = new ArrayList<>(); + int size = between(2, 5); + for (int i = 0; i < size; i++) { + nodes.add(dummyNode(randomBoolean(), randomBoolean(), randomBoolean())); + } + assertEquals(nodes, NodeSelector.ANY.select(nodes)); } public void testNotMasterOnly() { @@ -37,10 +44,11 @@ public void testNotMasterOnly() { Node masterAndData = dummyNode(true, true, randomBoolean()); Node client = dummyNode(false, false, randomBoolean()); Node data = dummyNode(false, true, randomBoolean()); - assertFalse(NodeSelector.NOT_MASTER_ONLY.select(masterOnly)); - assertTrue(NodeSelector.NOT_MASTER_ONLY.select(masterAndData)); - assertTrue(NodeSelector.NOT_MASTER_ONLY.select(client)); - assertTrue(NodeSelector.NOT_MASTER_ONLY.select(data)); + List nodes = Arrays.asList(masterOnly, masterAndData, client, data); + Collections.shuffle(nodes, getRandom()); + List expected = new ArrayList<>(nodes); + expected.remove(masterOnly); + assertEquals(expected, NodeSelector.NOT_MASTER_ONLY.select(nodes)); } private Node dummyNode(boolean master, boolean data, boolean ingest) { diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java index 03d308c3dc3a7..4ad9484365b12 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsIntegTests.java @@ -34,11 +34,13 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import static java.util.Collections.singletonList; import static org.elasticsearch.client.RestClientTestUtil.getAllStatusCodes; import static org.elasticsearch.client.RestClientTestUtil.randomErrorNoRetryStatusCode; import static org.elasticsearch.client.RestClientTestUtil.randomOkStatusCode; @@ -280,8 +282,13 @@ Response getResponse() { private NodeSelector firstPositionNodeSelector() { return new NodeSelector() { @Override - public boolean select(Node node) { - return httpHosts[0] == node.getHost(); + public List select(List nodes) { + for (Node node : nodes) { + if (httpHosts[0] == node.getHost()) { + return singletonList(node); + } + } + return Collections.emptyList(); } }; } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java index 3e2a8ee3e528b..34a123b8dac75 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java @@ -44,17 +44,21 @@ import java.io.IOException; import java.net.SocketTimeoutException; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import static java.util.Collections.singletonList; import static org.elasticsearch.client.RestClientTestUtil.randomErrorNoRetryStatusCode; import static org.elasticsearch.client.RestClientTestUtil.randomErrorRetryStatusCode; import static org.elasticsearch.client.RestClientTestUtil.randomHttpMethod; import static org.elasticsearch.client.RestClientTestUtil.randomOkStatusCode; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.hasItem; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -306,8 +310,9 @@ public void testRoundRobinRetryErrors() throws IOException { public void testWithNodeSelector() throws IOException { NodeSelector firstPositionOnly = new NodeSelector() { @Override - public boolean select(Node node) { - return nodes[0] == node; + public List select(List restClientNodes) { + assertThat(restClientNodes, hasItem(nodes[0])); + return singletonList(nodes[0]); } }; RestClientActions withNodeSelector = restClient.withNodeSelector(firstPositionOnly); diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java index 3fae90625af64..0c4a8e05dc160 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java @@ -22,25 +22,22 @@ import org.apache.http.Header; import org.apache.http.HttpHost; import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; -import org.elasticsearch.client.RestClient.NextHostsResult; import org.elasticsearch.client.RestClient.NodeTuple; import java.io.IOException; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; -import static org.hamcrest.Matchers.containsInAnyOrder; +import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; @@ -159,70 +156,94 @@ public void testBuildUriLeavesPathUntouched() { } } - public void testNextHostsOneTime() { + public void testSelectHosts() throws IOException { int iterations = 1000; HttpHost h1 = new HttpHost("1"); HttpHost h2 = new HttpHost("2"); HttpHost h3 = new HttpHost("3"); - Set nodes = new HashSet<>(); - nodes.add(new Node(h1, null, "1", null)); - nodes.add(new Node(h2, null, "2", null)); - nodes.add(new Node(h3, null, "3", null)); + List nodes = Arrays.asList( + new Node(h1, null, "1", null), + new Node(h2, null, "2", null), + new Node(h3, null, "3", null)); NodeSelector not1 = new NodeSelector() { @Override - public boolean select(Node node) { - return false == "1".equals(node.getVersion()); + public List select(List nodes) { + List result = new ArrayList<>(); + for (Node node : nodes) { + if (false == "1".equals(node.getVersion())) { + result.add(node); + } + } + return result; + } + + @Override + public String toString() { + return "NOT 1"; } }; NodeSelector noNodes = new NodeSelector() { @Override - public boolean select(Node node) { - return false; + public List select(List nodes) { + return Collections.emptyList(); + } + + @Override + public String toString() { + return "NONE"; } }; - NodeTuple> nodeTuple = new NodeTuple<>(nodes, null); + NodeTuple> nodeTuple = new NodeTuple<>(nodes, null); Map blacklist = new HashMap<>(); AtomicInteger lastNodeIndex = new AtomicInteger(0); long now = 0; // Normal case - NextHostsResult result = RestClient.nextHostsOneTime(nodeTuple, blacklist, - lastNodeIndex, now, NodeSelector.ANY); - assertThat(result.hosts, containsInAnyOrder(h1, h2, h3)); - List expectedHosts = new ArrayList<>(result.hosts); + List expectedHosts = Arrays.asList(h1, h2, h3); + assertEquals(expectedHosts, RestClient.selectHosts(nodeTuple, blacklist, + lastNodeIndex, now, NodeSelector.ANY)); // Calling it again rotates the set of results for (int i = 0; i < iterations; i++) { Collections.rotate(expectedHosts, 1); - assertEquals(expectedHosts, RestClient.nextHostsOneTime(nodeTuple, blacklist, - lastNodeIndex, now, NodeSelector.ANY).hosts); + assertEquals(expectedHosts, RestClient.selectHosts(nodeTuple, blacklist, + lastNodeIndex, now, NodeSelector.ANY)); } // Exclude some node lastNodeIndex.set(0); - result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, now, not1); - assertThat(result.hosts, containsInAnyOrder(h2, h3)); // h1 excluded - assertEquals(0, result.blacklisted); - assertEquals(1, result.selectorRejected); - assertEquals(0, result.selectorBlockedRevival); - expectedHosts = new ArrayList<>(result.hosts); + // h1 excluded + assertEquals(Arrays.asList(h2, h3), RestClient.selectHosts(nodeTuple, blacklist, + lastNodeIndex, now, not1)); // Calling it again rotates the set of results - for (int i = 0; i < iterations; i++) { - Collections.rotate(expectedHosts, 1); - assertEquals(expectedHosts, RestClient.nextHostsOneTime(nodeTuple, blacklist, - lastNodeIndex, now, not1).hosts); - } + assertEquals(Arrays.asList(h3, h2), RestClient.selectHosts(nodeTuple, blacklist, + lastNodeIndex, now, not1)); + // And again, same + assertEquals(Arrays.asList(h2, h3), RestClient.selectHosts(nodeTuple, blacklist, + lastNodeIndex, now, not1)); + /* + * But this time it doesn't because the list being filtered changes + * from (h1, h2, h3) to (h2, h3, h1) which both look the same when + * you filter out h1. + */ + assertEquals(Arrays.asList(h2, h3), RestClient.selectHosts(nodeTuple, blacklist, + lastNodeIndex, now, not1)); /* * Try a NodeSelector that excludes all nodes. This should - * return a failure. + * throw an exception */ - result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, now, noNodes); - assertNull(result.hosts); - assertEquals(0, result.blacklisted); - assertEquals(3, result.selectorRejected); - assertEquals(0, result.selectorBlockedRevival); + lastNodeIndex.set(0); + try { + RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, noNodes); + fail("expected selectHosts to fail"); + } catch (IOException e) { + String message = "NodeSelector [NONE] rejcted all nodes, living [" + + "[host=http://1, version=1], [host=http://2, version=2], " + + "[host=http://3, version=3]] and dead []"; + assertEquals(message, e.getMessage()); + } /* * Mark all nodes as dead and look up at a time *after* the @@ -233,18 +254,14 @@ public boolean select(Node node) { blacklist.put(h3, new DeadHostState(1, 3)); lastNodeIndex.set(0); now = 1000; - result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, - now, NodeSelector.ANY); - assertThat(result.hosts, containsInAnyOrder(h1, h2, h3)); - assertEquals(0, result.blacklisted); - assertEquals(0, result.selectorRejected); - assertEquals(0, result.selectorBlockedRevival); - expectedHosts = new ArrayList<>(result.hosts); + expectedHosts = Arrays.asList(h1, h2, h3); + assertEquals(expectedHosts, RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, + now, NodeSelector.ANY)); // Calling it again rotates the set of results for (int i = 0; i < iterations; i++) { Collections.rotate(expectedHosts, 1); - assertEquals(expectedHosts, RestClient.nextHostsOneTime(nodeTuple, blacklist, - lastNodeIndex, now, NodeSelector.ANY).hosts); + assertEquals(expectedHosts, RestClient.selectHosts(nodeTuple, blacklist, + lastNodeIndex, now, NodeSelector.ANY)); } /* @@ -252,24 +269,16 @@ public boolean select(Node node) { * Only the node closest to revival should come back. */ now = 0; - result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, - now, NodeSelector.ANY); - assertEquals(Collections.singleton(h1), result.hosts); - assertEquals(3, result.blacklisted); - assertEquals(0, result.selectorRejected); - assertEquals(0, result.selectorBlockedRevival); + assertEquals(singletonList(h1), RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, + now, NodeSelector.ANY)); /* * Now try with the nodes dead and *not* past their dead time * *and* a node selector that removes the node that is closest * to being revived. The second closest node should come back. */ - result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, - now, not1); - assertEquals(Collections.singleton(h2), result.hosts); - assertEquals(3, result.blacklisted); - assertEquals(0, result.selectorRejected); - assertEquals(1, result.selectorBlockedRevival); + assertEquals(singletonList(h2), RestClient.selectHosts(nodeTuple, blacklist, + lastNodeIndex, now, not1)); /* * Try a NodeSelector that excludes all nodes. This should @@ -277,11 +286,16 @@ public boolean select(Node node) { * because it'll block revival rather than outright reject * healthy nodes. */ - result = RestClient.nextHostsOneTime(nodeTuple, blacklist, lastNodeIndex, now, noNodes); - assertNull(result.hosts); - assertEquals(3, result.blacklisted); - assertEquals(0, result.selectorRejected); - assertEquals(3, result.selectorBlockedRevival); + lastNodeIndex.set(0); + try { + RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, noNodes); + fail("expected selectHosts to fail"); + } catch (IOException e) { + String message = "NodeSelector [NONE] rejcted all nodes, living [] and dead [" + + "[host=http://1, version=1], [host=http://2, version=2], " + + "[host=http://3, version=3]]"; + assertEquals(message, e.getMessage()); + } } private static RestClient createRestClient() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java index 586fbf3e56e8a..7c7147434e3f4 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java @@ -133,7 +133,7 @@ public static DoSection parse(XContentParser parser) throws IOException { } else if (token.isValue()) { NodeSelector newSelector = buildNodeSelector( parser.getTokenLocation(), selectorName, parser.text()); - nodeSelector = new NodeSelector.And(nodeSelector, newSelector); + nodeSelector = new NodeSelector.Compose(nodeSelector, newSelector); } } } else if (currentFieldName != null) { // must be part of API call then @@ -360,13 +360,19 @@ private static NodeSelector buildNodeSelector(XContentLocation location, String Version[] range = SkipSection.parseVersionRange(value); return new NodeSelector() { @Override - public boolean select(Node node) { - if (node.getVersion() == null) { - throw new IllegalStateException("expected [version] metadata to be set but got " - + node); + public List select(List nodes) { + List result = new ArrayList<>(nodes.size()); + for (Node node : nodes) { + if (node.getVersion() == null) { + throw new IllegalStateException("expected [version] metadata to be set but got " + + node); + } + Version version = Version.fromString(node.getVersion()); + if (version.onOrAfter(range[0]) && version.onOrBefore(range[1])) { + result.add(node); + } } - Version version = Version.fromString(node.getVersion()); - return version.onOrAfter(range[0]) && version.onOrBefore(range[1]); + return result; } @Override diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java index 6d00ddb7cb08a..588f5d17057b7 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java @@ -33,6 +33,7 @@ import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.Map; import static java.util.Collections.emptyList; @@ -518,12 +519,12 @@ public void testNodeSelector() throws IOException { DoSection doSection = DoSection.parse(parser); assertNotSame(NodeSelector.ANY, doSection.getApiCallSection().getNodeSelector()); - assertTrue(doSection.getApiCallSection().getNodeSelector() - .select(nodeWithVersion("5.2.1"))); - assertFalse(doSection.getApiCallSection().getNodeSelector() - .select(nodeWithVersion("6.1.2"))); - assertFalse(doSection.getApiCallSection().getNodeSelector() - .select(nodeWithVersion("1.7.0"))); + Node v170 = nodeWithVersion("1.7.0"); + Node v521 = nodeWithVersion("5.2.1"); + Node v550 = nodeWithVersion("5.5.0"); + Node v612 = nodeWithVersion("6.1.2"); + assertEquals(Arrays.asList(v521, v550), doSection.getApiCallSection().getNodeSelector() + .select(Arrays.asList(v170, v521, v550, v612))); ClientYamlTestExecutionContext context = mock(ClientYamlTestExecutionContext.class); ClientYamlTestResponse mockResponse = mock(ClientYamlTestResponse.class); when(context.callApi("indices.get_field_mapping", singletonMap("index", "test_index"), From ff5819ae6d86088efaa0ba9e07a08ff5bc30e5d1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 27 Mar 2018 15:21:59 -0400 Subject: [PATCH 22/27] Unused import --- .../src/main/java/org/elasticsearch/client/RestClientView.java | 1 - 1 file changed, 1 deletion(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java index 87b78024646e0..2639443819be6 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClientView.java @@ -24,7 +24,6 @@ import org.apache.http.Header; import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; /** * Stateless view into a {@link RestClient} with customized parameters. From 0704e1d980059c3bd3f96a0c0c5f5213677ae4e7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 27 Mar 2018 17:03:52 -0400 Subject: [PATCH 23/27] Spelling --- .../elasticsearch/client/NodeSelector.java | 11 ++++++++++- .../org/elasticsearch/client/RestClient.java | 2 +- .../elasticsearch/client/RestClientTests.java | 4 ++-- .../rest-api-spec/test/README.asciidoc | 19 +++++++++++++++++++ 4 files changed, 32 insertions(+), 4 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java index cea3886a32333..4b20a97b2df3c 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java +++ b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java @@ -29,7 +29,16 @@ */ public interface NodeSelector { /** - * Select the {@link Node}s to which to send requests. + * Select the {@link Node}s to which to send requests. This may be called + * twice per request, once for "living" nodes that have not had been + * blacklisted by previous errors if there are any. If it returns an + * empty list when sent the living nodes or if there aren't any living + * nodes left then this will be called with a list of "dead" nodes that + * have been blacklisted by previous failures. In both cases it should + * return a list of nodes sorted by its preference for which node is used. + * If it is operating on "living" nodes that it returns function as + * fallbacks in case of request failures. If it is operating on dead nodes + * then the dead node that it returns is attempted but no others. * * @param nodes an unmodifiable list of {@linkplain Node}s in the order * that the {@link RestClient} would prefer to use them diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 76a3e9ec167ed..7ea04a5b37e06 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -451,7 +451,7 @@ public int compare(DeadNodeAndRevival lhs, DeadNodeAndRevival rhs) { return singletonList(selectedDeadNodes.get(0).getHost()); } } - throw new IOException("NodeSelector [" + nodeSelector + "] rejcted all nodes, " + throw new IOException("NodeSelector [" + nodeSelector + "] rejected all nodes, " + "living " + livingNodes + " and dead " + deadNodes); } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java index 0c4a8e05dc160..b5f34f2ae3eba 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java @@ -239,7 +239,7 @@ public String toString() { RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, noNodes); fail("expected selectHosts to fail"); } catch (IOException e) { - String message = "NodeSelector [NONE] rejcted all nodes, living [" + String message = "NodeSelector [NONE] rejected all nodes, living [" + "[host=http://1, version=1], [host=http://2, version=2], " + "[host=http://3, version=3]] and dead []"; assertEquals(message, e.getMessage()); @@ -291,7 +291,7 @@ public String toString() { RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, noNodes); fail("expected selectHosts to fail"); } catch (IOException e) { - String message = "NodeSelector [NONE] rejcted all nodes, living [] and dead [" + String message = "NodeSelector [NONE] rejected all nodes, living [] and dead [" + "[host=http://1, version=1], [host=http://2, version=2], " + "[host=http://3, version=3]]"; assertEquals(message, e.getMessage()); diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc index c93873a5be429..75135881b8353 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc @@ -198,6 +198,25 @@ header. The warnings must match exactly. Using it looks like this: .... +If the arguments to `do` include `node_selector` then the request is only +sent to nodes that match the `node_selector`. Currently only the `version` +selector is supported and it has the same logic as the `version` field in +`skip`. It looks like this: + +.... +"test id": + - skip: + features: node_selector + - do: + node_selector: + version: " - 6.9.99" + index: + index: test-weird-index-中文 + type: weird.type + id: 1 + body: { foo: bar } +.... + === `set` For some tests, it is necessary to extract a value from the previous `response`, in From b8eb23f4c98a3acd3052b274fa14ea2d77d2a927 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 28 Mar 2018 16:19:50 -0400 Subject: [PATCH 24/27] Fix 2.x support And add tests that parse the `/_nodes/http` API responses from older versions so we verify that we don't break it. --- .../java/org/elasticsearch/client/Node.java | 40 +++-- .../client/NodeSelectorTests.java | 5 +- .../org/elasticsearch/client/NodeTests.java | 38 ++-- .../client/RestClientMultipleHostsTests.java | 2 +- .../elasticsearch/client/RestClientTests.java | 6 +- .../sniff/ElasticsearchHostsSniffer.java | 55 +++++- ...icsearchHostsSnifferParseExampleTests.java | 112 ++++++++++++ .../sniff/ElasticsearchHostsSnifferTests.java | 6 +- .../client/sniff/MockHostsSniffer.java | 2 +- .../src/test/resources/2.0.0_nodes_http.json | 141 +++++++++++++++ .../src/test/resources/5.0.0_nodes_http.json | 169 ++++++++++++++++++ .../src/test/resources/6.0.0_nodes_http.json | 169 ++++++++++++++++++ client/sniffer/src/test/resources/readme.txt | 4 + .../yaml/ESClientYamlSuiteTestCaseTests.java | 22 ++- .../rest/yaml/section/DoSectionTests.java | 2 +- 15 files changed, 722 insertions(+), 51 deletions(-) create mode 100644 client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferParseExampleTests.java create mode 100644 client/sniffer/src/test/resources/2.0.0_nodes_http.json create mode 100644 client/sniffer/src/test/resources/5.0.0_nodes_http.json create mode 100644 client/sniffer/src/test/resources/6.0.0_nodes_http.json create mode 100644 client/sniffer/src/test/resources/readme.txt diff --git a/client/rest/src/main/java/org/elasticsearch/client/Node.java b/client/rest/src/main/java/org/elasticsearch/client/Node.java index 57362f88c41de..5345ba59bd24e 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/Node.java +++ b/client/rest/src/main/java/org/elasticsearch/client/Node.java @@ -19,11 +19,11 @@ package org.elasticsearch.client; -import static java.util.Collections.unmodifiableList; +import static java.util.Collections.unmodifiableSet; -import java.util.ArrayList; -import java.util.List; +import java.util.HashSet; import java.util.Objects; +import java.util.Set; import org.apache.http.HttpHost; @@ -40,7 +40,11 @@ public class Node { * around because they allow you to find a host based on any address it * is listening on. */ - private final List boundHosts; + private final Set boundHosts; + /** + * Name of the node as configured by the {@code node.name} attribute. + */ + private final String name; /** * Version of Elasticsearch that the node is running or {@code null} * if we don't know the version. @@ -57,12 +61,13 @@ public class Node { * {@code host} are nullable and implementations of {@link NodeSelector} * need to decide what to do in their absence. */ - public Node(HttpHost host, List boundHosts, String version, Roles roles) { + public Node(HttpHost host, Set boundHosts, String name, String version, Roles roles) { if (host == null) { throw new IllegalArgumentException("host cannot be null"); } this.host = host; this.boundHosts = boundHosts; + this.name = name; this.version = version; this.roles = roles; } @@ -71,7 +76,7 @@ public Node(HttpHost host, List boundHosts, String version, Roles role * Create a {@linkplain Node} without any metadata. */ public Node(HttpHost host) { - this(host, null, null, null); + this(host, null, null, null, null); } /** @@ -85,13 +90,13 @@ public Node withHost(HttpHost host) { * If the new host isn't in the bound hosts list we add it so the * result looks sane. */ - List boundHosts = this.boundHosts; + Set boundHosts = this.boundHosts; if (false == boundHosts.contains(host)) { - boundHosts = new ArrayList<>(boundHosts); + boundHosts = new HashSet<>(boundHosts); boundHosts.add(host); - boundHosts = unmodifiableList(boundHosts); + boundHosts = unmodifiableSet(boundHosts); } - return new Node(host, boundHosts, version, roles); + return new Node(host, boundHosts, name, version, roles); } /** @@ -106,10 +111,17 @@ public HttpHost getHost() { * around because they allow you to find a host based on any address it * is listening on. */ - public List getBoundHosts() { + public Set getBoundHosts() { return boundHosts; } + /** + * @return the name + */ + public String getName() { + return name; + } + /** * Version of Elasticsearch that the node is running or {@code null} * if we don't know the version. @@ -133,6 +145,9 @@ public String toString() { if (boundHosts != null) { b.append(", bound=").append(boundHosts); } + if (name != null) { + b.append(", name=").append(name); + } if (version != null) { b.append(", version=").append(version); } @@ -151,12 +166,13 @@ public boolean equals(Object obj) { return host.equals(other.host) && Objects.equals(boundHosts, other.boundHosts) && Objects.equals(version, other.version) + && Objects.equals(name, other.name) && Objects.equals(roles, other.roles); } @Override public int hashCode() { - return Objects.hash(host, boundHosts, version, roles); + return Objects.hash(host, boundHosts, name, version, roles); } /** diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java index dfbcd9461c580..a8f5c70c33fe8 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java @@ -52,7 +52,8 @@ public void testNotMasterOnly() { } private Node dummyNode(boolean master, boolean data, boolean ingest) { - return new Node(new HttpHost("dummy"), Collections.emptyList(), - randomAsciiAlphanumOfLength(5), new Roles(master, data, ingest)); + return new Node(new HttpHost("dummy"), Collections.emptySet(), + randomAsciiAlphanumOfLength(5), randomAsciiAlphanumOfLength(5), + new Roles(master, data, ingest)); } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java index c328ef7fa5584..5ea1eb8268a0b 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java @@ -23,8 +23,9 @@ import org.elasticsearch.client.Node.Roles; import java.util.Arrays; +import java.util.HashSet; -import static java.util.Collections.singletonList; +import static java.util.Collections.singleton; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -35,8 +36,9 @@ public void testWithHost() { HttpHost h2 = new HttpHost("2"); HttpHost h3 = new HttpHost("3"); - Node n = new Node(h1, Arrays.asList(h1, h2), randomAsciiAlphanumOfLength(5), - new Roles(randomBoolean(), randomBoolean(), randomBoolean())); + Node n = new Node(h1, new HashSet<>(Arrays.asList(h1, h2)), + randomAsciiAlphanumOfLength(5), randomAsciiAlphanumOfLength(5), + new Roles(randomBoolean(), randomBoolean(), randomBoolean())); // Host is in nthe bound hosts list assertEquals(h2, n.withHost(h2).getHost()); @@ -44,36 +46,44 @@ public void testWithHost() { // Host not in the bound hosts list assertEquals(h3, n.withHost(h3).getHost()); - assertEquals(Arrays.asList(h1, h2, h3), n.withHost(h3).getBoundHosts()); + assertEquals(new HashSet<>(Arrays.asList(h1, h2, h3)), n.withHost(h3).getBoundHosts()); } public void testToString() { assertEquals("[host=http://1]", new Node(new HttpHost("1")).toString()); assertEquals("[host=http://1, roles=mdi]", new Node(new HttpHost("1"), - null, null, new Roles(true, true, true)).toString()); + null, null, null, new Roles(true, true, true)).toString()); assertEquals("[host=http://1, version=ver]", new Node(new HttpHost("1"), - null, "ver", null).toString()); + null, null, "ver", null).toString()); + assertEquals("[host=http://1, name=nam]", new Node(new HttpHost("1"), + null, "nam", null, null).toString()); assertEquals("[host=http://1, bound=[http://1, http://2]]", new Node(new HttpHost("1"), - Arrays.asList(new HttpHost("1"), new HttpHost("2")), null, null).toString()); + new HashSet<>(Arrays.asList(new HttpHost("1"), new HttpHost("2"))), null, null, null).toString()); + assertEquals("[host=http://1, bound=[http://1, http://2], name=nam, version=ver, roles=m]", + new Node(new HttpHost("1"), new HashSet<>(Arrays.asList(new HttpHost("1"), new HttpHost("2"))), + "nam", "ver", new Roles(true, false, false)).toString()); + } public void testEqualsAndHashCode() { HttpHost host = new HttpHost(randomAsciiAlphanumOfLength(5)); Node node = new Node(host, - randomBoolean() ? null : singletonList(host), + randomBoolean() ? null : singleton(host), + randomBoolean() ? null : randomAsciiAlphanumOfLength(5), randomBoolean() ? null : randomAsciiAlphanumOfLength(5), randomBoolean() ? null : new Roles(true, true, true)); assertFalse(node.equals(null)); assertTrue(node.equals(node)); assertEquals(node.hashCode(), node.hashCode()); - Node copy = new Node(host, node.getBoundHosts(), node.getVersion(), node.getRoles()); + Node copy = new Node(host, node.getBoundHosts(), node.getName(), node.getVersion(), node.getRoles()); assertTrue(node.equals(copy)); assertEquals(node.hashCode(), copy.hashCode()); assertFalse(node.equals(new Node(new HttpHost(host.toHostString() + "changed"), node.getBoundHosts(), - node.getVersion(), node.getRoles()))); - assertFalse(node.equals(new Node(host, Arrays.asList(host, new HttpHost(host.toHostString() + "changed")), - node.getVersion(), node.getRoles()))); - assertFalse(node.equals(new Node(host, node.getBoundHosts(), node.getVersion() + "changed", node.getRoles()))); - assertFalse(node.equals(new Node(host, node.getBoundHosts(), node.getVersion(), new Roles(false, false, false)))); + node.getName(), node.getVersion(), node.getRoles()))); + assertFalse(node.equals(new Node(host, new HashSet<>(Arrays.asList(host, new HttpHost(host.toHostString() + "changed"))), + node.getName(), node.getVersion(), node.getRoles()))); + assertFalse(node.equals(new Node(host, node.getBoundHosts(), node.getName() + "changed", node.getVersion(), node.getRoles()))); + assertFalse(node.equals(new Node(host, node.getBoundHosts(), node.getName(), node.getVersion() + "changed", node.getRoles()))); + assertFalse(node.equals(new Node(host, node.getBoundHosts(), node.getName(), node.getVersion(), new Roles(false, false, false)))); } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java index 34a123b8dac75..03212a700a186 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientMultipleHostsTests.java @@ -331,7 +331,7 @@ public void testSetNodes() throws IOException { Node[] newNodes = new Node[nodes.length]; for (int i = 0; i < nodes.length; i++) { Roles roles = i == 0 ? new Roles(false, true, true) : new Roles(true, false, false); - newNodes[i] = new Node(nodes[i].getHost(), null, null, roles); + newNodes[i] = new Node(nodes[i].getHost(), null, null, null, roles); } restClient.setNodes(newNodes); RestClientActions withNodeSelector = restClient.withNodeSelector(NodeSelector.NOT_MASTER_ONLY); diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java index b5f34f2ae3eba..590c15e5a9362 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java @@ -162,9 +162,9 @@ public void testSelectHosts() throws IOException { HttpHost h2 = new HttpHost("2"); HttpHost h3 = new HttpHost("3"); List nodes = Arrays.asList( - new Node(h1, null, "1", null), - new Node(h2, null, "2", null), - new Node(h3, null, "3", null)); + new Node(h1, null, null, "1", null), + new Node(h2, null, null, "2", null), + new Node(h3, null, null, "3", null)); NodeSelector not1 = new NodeSelector() { @Override diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java index 1de6fb468a9e6..4ee57692feb58 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java @@ -36,10 +36,11 @@ import java.net.URI; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; /** @@ -95,10 +96,10 @@ public ElasticsearchHostsSniffer(RestClient restClient, long sniffRequestTimeout @Override public List sniffHosts() throws IOException { Response response = restClient.performRequest("get", "/_nodes/http", sniffRequestParams); - return readHosts(response.getEntity()); + return readHosts(response.getEntity(), scheme, jsonFactory); } - private List readHosts(HttpEntity entity) throws IOException { + static List readHosts(HttpEntity entity, Scheme scheme, JsonFactory jsonFactory) throws IOException { try (InputStream inputStream = entity.getContent()) { JsonParser parser = jsonFactory.createParser(inputStream); if (parser.nextToken() != JsonToken.START_OBJECT) { @@ -131,13 +132,19 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Li * test framework where we sometimes publish ipv6 addresses but the * tests contact the node on ipv4. */ - List boundHosts = new ArrayList<>(); - String fieldName = null; + Set boundHosts = new HashSet<>(); + String name = null; String version = null; + String fieldName = null; + // Used to read roles from 5.0+ boolean sawRoles = false; boolean master = false; boolean data = false; boolean ingest = false; + // Used to read roles from 2.x + Boolean masterAttribute = null; + Boolean dataAttribute = null; + boolean clientAttribute = false; while (parser.nextToken() != JsonToken.END_OBJECT) { if (parser.getCurrentToken() == JsonToken.FIELD_NAME) { fieldName = parser.getCurrentName(); @@ -158,6 +165,18 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Li parser.skipChildren(); } } + } else if ("attributes".equals(fieldName)) { + while (parser.nextToken() != JsonToken.END_OBJECT) { + if (parser.getCurrentToken() == JsonToken.VALUE_STRING && "master".equals(parser.getCurrentName())) { + masterAttribute = toBoolean(parser.getValueAsString()); + } else if (parser.getCurrentToken() == JsonToken.VALUE_STRING && "data".equals(parser.getCurrentName())) { + dataAttribute = toBoolean(parser.getValueAsString()); + } else if (parser.getCurrentToken() == JsonToken.VALUE_STRING && "client".equals(parser.getCurrentName())) { + clientAttribute = toBoolean(parser.getValueAsString()); + } else if (parser.getCurrentToken() == JsonToken.START_OBJECT) { + parser.skipChildren(); + } + } } else { parser.skipChildren(); } @@ -185,6 +204,8 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Li } else if (parser.currentToken().isScalarValue()) { if ("version".equals(fieldName)) { version = parser.getText(); + } else if ("name".equals(fieldName)) { + name = parser.getText(); } } } @@ -193,10 +214,19 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Li logger.debug("skipping node [" + nodeId + "] with http disabled"); } else { logger.trace("adding node [" + nodeId + "]"); - assert sawRoles : "didn't see roles for [" + nodeId + "]"; + if (version.startsWith("2.")) { + /* + * 2.x doesn't send roles, instead we try to read them from + * attributes. + */ + master = masterAttribute == null ? false == clientAttribute : masterAttribute; + data = dataAttribute == null ? false == clientAttribute : dataAttribute; + } else { + assert sawRoles : "didn't see roles for [" + nodeId + "]"; + } assert boundHosts.contains(publishedHost) : "[" + nodeId + "] doesn't make sense! publishedHost should be in boundHosts"; - nodes.add(new Node(publishedHost, boundHosts, version, new Roles(master, data, ingest))); + nodes.add(new Node(publishedHost, boundHosts, name, version, new Roles(master, data, ingest))); } } @@ -214,4 +244,15 @@ public String toString() { return name; } } + + private static boolean toBoolean(String string) { + switch (string) { + case "true": + return true; + case "false": + return false; + default: + throw new IllegalArgumentException("[" + string + "] is not a valid boolean"); + } + } } diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferParseExampleTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferParseExampleTests.java new file mode 100644 index 0000000000000..41156d9b58863 --- /dev/null +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferParseExampleTests.java @@ -0,0 +1,112 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.sniff; + +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.InputStreamEntity; +import org.elasticsearch.client.Node; +import org.elasticsearch.client.RestClientTestCase; +import org.elasticsearch.client.Node.Roles; +import org.elasticsearch.client.sniff.ElasticsearchHostsSniffer.Scheme; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import com.fasterxml.jackson.core.JsonFactory; + +import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.hasSize; +import static org.junit.Assert.assertThat; + +/** + * Test parsing the response from the {@code /_nodes/http} API from fixed + * versions of Elasticsearch. + */ +public class ElasticsearchHostsSnifferParseExampleTests extends RestClientTestCase { + private void checkFile(String file, Node... expected) throws IOException { + InputStream in = Thread.currentThread().getContextClassLoader().getResourceAsStream(file); + if (in == null) { + throw new IllegalArgumentException("Couldn't find [" + file + "]"); + } + try { + HttpEntity entity = new InputStreamEntity(in, ContentType.APPLICATION_JSON); + List nodes = ElasticsearchHostsSniffer.readHosts(entity, Scheme.HTTP, new JsonFactory()); + // Use these assertions because the error messages are nicer than hasItems. + assertThat(nodes, hasSize(expected.length)); + for (Node expectedNode : expected) { + assertThat(nodes, hasItem(expectedNode)); + } + } finally { + in.close(); + } + } + + public void test2x() throws IOException { + checkFile("2.0.0_nodes_http.json", + node(9200, "m1", "2.0.0", true, false, false), + node(9202, "m2", "2.0.0", true, true, false), + node(9201, "m3", "2.0.0", true, false, false), + node(9205, "d1", "2.0.0", false, true, false), + node(9204, "d2", "2.0.0", false, true, false), + node(9203, "d3", "2.0.0", false, true, false), + node(9207, "c1", "2.0.0", false, false, false), + node(9206, "c2", "2.0.0", false, false, false)); + } + + public void test5x() throws IOException { + checkFile("5.0.0_nodes_http.json", + node(9200, "m1", "5.0.0", true, false, true), + node(9201, "m2", "5.0.0", true, true, true), + node(9202, "m3", "5.0.0", true, false, true), + node(9203, "d1", "5.0.0", false, true, true), + node(9204, "d2", "5.0.0", false, true, true), + node(9205, "d3", "5.0.0", false, true, true), + node(9206, "c1", "5.0.0", false, false, true), + node(9207, "c2", "5.0.0", false, false, true)); + } + + public void test6x() throws IOException { + checkFile("6.0.0_nodes_http.json", + node(9200, "m1", "6.0.0", true, false, true), + node(9201, "m2", "6.0.0", true, true, true), + node(9202, "m3", "6.0.0", true, false, true), + node(9203, "d1", "6.0.0", false, true, true), + node(9204, "d2", "6.0.0", false, true, true), + node(9205, "d3", "6.0.0", false, true, true), + node(9206, "c1", "6.0.0", false, false, true), + node(9207, "c2", "6.0.0", false, false, true)); + } + + private Node node(int port, String name, String version, boolean master, boolean data, boolean ingest) { + HttpHost host = new HttpHost("127.0.0.1", port); + Set boundHosts = new HashSet<>(2); + boundHosts.add(host); + boundHosts.add(new HttpHost("[::1]", port)); + return new Node(host, boundHosts, name, version, new Roles(master, data, ingest)); + } +} diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java index 8a68aab595b4c..0f3501b55311f 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java @@ -50,7 +50,6 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -196,7 +195,7 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme String host = "host" + i; int port = RandomNumbers.randomIntBetween(getRandom(), 9200, 9299); HttpHost publishHost = new HttpHost(host, port, scheme.toString()); - List boundHosts = new ArrayList<>(); + Set boundHosts = new HashSet<>(); boundHosts.add(publishHost); if (randomBoolean()) { @@ -207,6 +206,7 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme } Node node = new Node(publishHost, boundHosts, randomAsciiAlphanumOfLength(5), + randomAsciiAlphanumOfLength(5), new Node.Roles(randomBoolean(), randomBoolean(), randomBoolean())); generator.writeObjectFieldStart(nodeId); @@ -258,6 +258,8 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme generator.writeFieldName("version"); generator.writeString(node.getVersion()); + generator.writeFieldName("name"); + generator.writeString(node.getName()); int numAttributes = RandomNumbers.randomIntBetween(getRandom(), 0, 3); Map attributes = new HashMap<>(numAttributes); diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java index 6bad7ee1af6ee..bb5b443ff0f9f 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/MockHostsSniffer.java @@ -33,7 +33,7 @@ class MockHostsSniffer implements HostsSniffer { @Override public List sniffHosts() throws IOException { return Collections.singletonList(new Node( - new HttpHost("localhost", 9200), Collections.emptyList(), + new HttpHost("localhost", 9200), Collections.emptySet(), "mock node name", "mock version", new Node.Roles(false, false, false))); } } diff --git a/client/sniffer/src/test/resources/2.0.0_nodes_http.json b/client/sniffer/src/test/resources/2.0.0_nodes_http.json new file mode 100644 index 0000000000000..b370e78e16011 --- /dev/null +++ b/client/sniffer/src/test/resources/2.0.0_nodes_http.json @@ -0,0 +1,141 @@ +{ + "cluster_name" : "elasticsearch", + "nodes" : { + "qYUZ_8bTRwODPxukDlFw6Q" : { + "name" : "d2", + "transport_address" : "127.0.0.1:9304", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "2.0.0", + "build" : "de54438", + "http_address" : "127.0.0.1:9204", + "attributes" : { + "master" : "false" + }, + "http" : { + "bound_address" : [ "127.0.0.1:9204", "[::1]:9204" ], + "publish_address" : "127.0.0.1:9204", + "max_content_length_in_bytes" : 104857600 + } + }, + "Yej5UVNgR2KgBjUFHOQpCw" : { + "name" : "c1", + "transport_address" : "127.0.0.1:9307", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "2.0.0", + "build" : "de54438", + "http_address" : "127.0.0.1:9207", + "attributes" : { + "data" : "false", + "master" : "false" + }, + "http" : { + "bound_address" : [ "127.0.0.1:9207", "[::1]:9207" ], + "publish_address" : "127.0.0.1:9207", + "max_content_length_in_bytes" : 104857600 + } + }, + "mHttJwhwReangKEx9EGuAg" : { + "name" : "m3", + "transport_address" : "127.0.0.1:9301", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "2.0.0", + "build" : "de54438", + "http_address" : "127.0.0.1:9201", + "attributes" : { + "data" : "false", + "master" : "true" + }, + "http" : { + "bound_address" : [ "127.0.0.1:9201", "[::1]:9201" ], + "publish_address" : "127.0.0.1:9201", + "max_content_length_in_bytes" : 104857600 + } + }, + "6Erdptt_QRGLxMiLi9mTkg" : { + "name" : "c2", + "transport_address" : "127.0.0.1:9306", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "2.0.0", + "build" : "de54438", + "http_address" : "127.0.0.1:9206", + "attributes" : { + "data" : "false", + "client" : "true" + }, + "http" : { + "bound_address" : [ "127.0.0.1:9206", "[::1]:9206" ], + "publish_address" : "127.0.0.1:9206", + "max_content_length_in_bytes" : 104857600 + } + }, + "mLRCZBypTiys6e8KY5DMnA" : { + "name" : "m1", + "transport_address" : "127.0.0.1:9300", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "2.0.0", + "build" : "de54438", + "http_address" : "127.0.0.1:9200", + "attributes" : { + "data" : "false" + }, + "http" : { + "bound_address" : [ "127.0.0.1:9200", "[::1]:9200" ], + "publish_address" : "127.0.0.1:9200", + "max_content_length_in_bytes" : 104857600 + } + }, + "pVqOhytXQwetsZVzCBppYw" : { + "name" : "m2", + "transport_address" : "127.0.0.1:9302", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "2.0.0", + "build" : "de54438", + "http_address" : "127.0.0.1:9202", + "http" : { + "bound_address" : [ "127.0.0.1:9202", "[::1]:9202" ], + "publish_address" : "127.0.0.1:9202", + "max_content_length_in_bytes" : 104857600 + } + }, + "ARyzVfpJSw2a9TOIUpbsBA" : { + "name" : "d1", + "transport_address" : "127.0.0.1:9305", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "2.0.0", + "build" : "de54438", + "http_address" : "127.0.0.1:9205", + "attributes" : { + "master" : "false" + }, + "http" : { + "bound_address" : [ "127.0.0.1:9205", "[::1]:9205" ], + "publish_address" : "127.0.0.1:9205", + "max_content_length_in_bytes" : 104857600 + } + }, + "2Hpid-g5Sc2BKCevhN6VQw" : { + "name" : "d3", + "transport_address" : "127.0.0.1:9303", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "2.0.0", + "build" : "de54438", + "http_address" : "127.0.0.1:9203", + "attributes" : { + "master" : "false" + }, + "http" : { + "bound_address" : [ "127.0.0.1:9203", "[::1]:9203" ], + "publish_address" : "127.0.0.1:9203", + "max_content_length_in_bytes" : 104857600 + } + } + } +} diff --git a/client/sniffer/src/test/resources/5.0.0_nodes_http.json b/client/sniffer/src/test/resources/5.0.0_nodes_http.json new file mode 100644 index 0000000000000..7a7d143ecaf43 --- /dev/null +++ b/client/sniffer/src/test/resources/5.0.0_nodes_http.json @@ -0,0 +1,169 @@ +{ + "_nodes" : { + "total" : 8, + "successful" : 8, + "failed" : 0 + }, + "cluster_name" : "test", + "nodes" : { + "DXz_rhcdSF2xJ96qyjaLVw" : { + "name" : "m1", + "transport_address" : "127.0.0.1:9300", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "5.0.0", + "build_hash" : "253032b", + "roles" : [ + "master", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9200", + "127.0.0.1:9200" + ], + "publish_address" : "127.0.0.1:9200", + "max_content_length_in_bytes" : 104857600 + } + }, + "53Mi6jYdRgeR1cdyuoNfQQ" : { + "name" : "m2", + "transport_address" : "127.0.0.1:9301", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "5.0.0", + "build_hash" : "253032b", + "roles" : [ + "master", + "data", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9201", + "127.0.0.1:9201" + ], + "publish_address" : "127.0.0.1:9201", + "max_content_length_in_bytes" : 104857600 + } + }, + "XBIghcHiRlWP9c4vY6rETw" : { + "name" : "c2", + "transport_address" : "127.0.0.1:9307", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "5.0.0", + "build_hash" : "253032b", + "roles" : [ + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9207", + "127.0.0.1:9207" + ], + "publish_address" : "127.0.0.1:9207", + "max_content_length_in_bytes" : 104857600 + } + }, + "cFM30FlyS8K1njH_bovwwQ" : { + "name" : "d1", + "transport_address" : "127.0.0.1:9303", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "5.0.0", + "build_hash" : "253032b", + "roles" : [ + "data", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9203", + "127.0.0.1:9203" + ], + "publish_address" : "127.0.0.1:9203", + "max_content_length_in_bytes" : 104857600 + } + }, + "eoVUVRGNRDyyOapqIcrsIA" : { + "name" : "d2", + "transport_address" : "127.0.0.1:9304", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "5.0.0", + "build_hash" : "253032b", + "roles" : [ + "data", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9204", + "127.0.0.1:9204" + ], + "publish_address" : "127.0.0.1:9204", + "max_content_length_in_bytes" : 104857600 + } + }, + "xPN76uDcTP-DyXaRzPg2NQ" : { + "name" : "c1", + "transport_address" : "127.0.0.1:9306", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "5.0.0", + "build_hash" : "253032b", + "roles" : [ + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9206", + "127.0.0.1:9206" + ], + "publish_address" : "127.0.0.1:9206", + "max_content_length_in_bytes" : 104857600 + } + }, + "RY0oW2d7TISEqazk-U4Kcw" : { + "name" : "d3", + "transport_address" : "127.0.0.1:9305", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "5.0.0", + "build_hash" : "253032b", + "roles" : [ + "data", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9205", + "127.0.0.1:9205" + ], + "publish_address" : "127.0.0.1:9205", + "max_content_length_in_bytes" : 104857600 + } + }, + "tU0rXEZmQ9GsWfn2TQ4kow" : { + "name" : "m3", + "transport_address" : "127.0.0.1:9302", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "5.0.0", + "build_hash" : "253032b", + "roles" : [ + "master", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9202", + "127.0.0.1:9202" + ], + "publish_address" : "127.0.0.1:9202", + "max_content_length_in_bytes" : 104857600 + } + } + } +} diff --git a/client/sniffer/src/test/resources/6.0.0_nodes_http.json b/client/sniffer/src/test/resources/6.0.0_nodes_http.json new file mode 100644 index 0000000000000..5a8905da64c89 --- /dev/null +++ b/client/sniffer/src/test/resources/6.0.0_nodes_http.json @@ -0,0 +1,169 @@ +{ + "_nodes" : { + "total" : 8, + "successful" : 8, + "failed" : 0 + }, + "cluster_name" : "test", + "nodes" : { + "FX9npqGQSL2mOGF8Zkf3hw" : { + "name" : "m2", + "transport_address" : "127.0.0.1:9301", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "6.0.0", + "build_hash" : "8f0685b", + "roles" : [ + "master", + "data", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9201", + "127.0.0.1:9201" + ], + "publish_address" : "127.0.0.1:9201", + "max_content_length_in_bytes" : 104857600 + } + }, + "jmUqzYLGTbWCg127kve3Tg" : { + "name" : "d1", + "transport_address" : "127.0.0.1:9303", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "6.0.0", + "build_hash" : "8f0685b", + "roles" : [ + "data", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9203", + "127.0.0.1:9203" + ], + "publish_address" : "127.0.0.1:9203", + "max_content_length_in_bytes" : 104857600 + } + }, + "soBU6bzvTOqdLxPstSbJ2g" : { + "name" : "d3", + "transport_address" : "127.0.0.1:9305", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "6.0.0", + "build_hash" : "8f0685b", + "roles" : [ + "data", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9205", + "127.0.0.1:9205" + ], + "publish_address" : "127.0.0.1:9205", + "max_content_length_in_bytes" : 104857600 + } + }, + "mtYDAhURTP6twdmNAkMnOg" : { + "name" : "m3", + "transport_address" : "127.0.0.1:9302", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "6.0.0", + "build_hash" : "8f0685b", + "roles" : [ + "master", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9202", + "127.0.0.1:9202" + ], + "publish_address" : "127.0.0.1:9202", + "max_content_length_in_bytes" : 104857600 + } + }, + "URxHiUQPROOt1G22Ev6lXw" : { + "name" : "c2", + "transport_address" : "127.0.0.1:9307", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "6.0.0", + "build_hash" : "8f0685b", + "roles" : [ + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9207", + "127.0.0.1:9207" + ], + "publish_address" : "127.0.0.1:9207", + "max_content_length_in_bytes" : 104857600 + } + }, + "_06S_kWoRqqFR8Z8CS3JRw" : { + "name" : "c1", + "transport_address" : "127.0.0.1:9306", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "6.0.0", + "build_hash" : "8f0685b", + "roles" : [ + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9206", + "127.0.0.1:9206" + ], + "publish_address" : "127.0.0.1:9206", + "max_content_length_in_bytes" : 104857600 + } + }, + "QZE5Bd6DQJmnfVs2dglOvA" : { + "name" : "d2", + "transport_address" : "127.0.0.1:9304", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "6.0.0", + "build_hash" : "8f0685b", + "roles" : [ + "data", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9204", + "127.0.0.1:9204" + ], + "publish_address" : "127.0.0.1:9204", + "max_content_length_in_bytes" : 104857600 + } + }, + "_3mTXg6dSweZn5ReB2fQqw" : { + "name" : "m1", + "transport_address" : "127.0.0.1:9300", + "host" : "127.0.0.1", + "ip" : "127.0.0.1", + "version" : "6.0.0", + "build_hash" : "8f0685b", + "roles" : [ + "master", + "ingest" + ], + "http" : { + "bound_address" : [ + "[::1]:9200", + "127.0.0.1:9200" + ], + "publish_address" : "127.0.0.1:9200", + "max_content_length_in_bytes" : 104857600 + } + } + } +} diff --git a/client/sniffer/src/test/resources/readme.txt b/client/sniffer/src/test/resources/readme.txt new file mode 100644 index 0000000000000..ad13fb3170046 --- /dev/null +++ b/client/sniffer/src/test/resources/readme.txt @@ -0,0 +1,4 @@ +`*_node_http.json` contains files created by spining up toy clusters with a +few nodes in different configurations locally at various versions. They are +for testing `ElasticsearchHostsSniffer` against different versions of +Elasticsearch. diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java index 825f05f78b0b6..cd01e355f5885 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java @@ -21,6 +21,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -31,6 +32,7 @@ import org.elasticsearch.test.ESTestCase; import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.greaterThan; @@ -111,16 +113,20 @@ public void testAttachSniffedMetadataOnClientOk() { }; List nodesWithMetadata = Arrays.asList(new Node[] { // This node matches exactly: - new Node(new HttpHost("1"), emptyList(), randomAlphaOfLength(5), randomRoles()), - // This node also matches exactly but has bound hosts which don't matter: - new Node(new HttpHost("2"), Arrays.asList(new HttpHost("2"), new HttpHost("not2")), + new Node(new HttpHost("1"), emptySet(), randomAlphaOfLength(5), randomAlphaOfLength(5), randomRoles()), + // This node also matches exactly but has bound hosts which don't matter: + new Node(new HttpHost("2"), + new HashSet<>(Arrays.asList(new HttpHost("2"), new HttpHost("not2"))), + randomAlphaOfLength(5), randomAlphaOfLength(5), randomRoles()), // This node's host doesn't match but one of its published hosts does so // we return a modified version of it: - new Node(new HttpHost("not3"), Arrays.asList(new HttpHost("not3"), new HttpHost("3")), - randomAlphaOfLength(5), randomRoles()), + new Node(new HttpHost("not3"), + new HashSet<>(Arrays.asList(new HttpHost("not3"), new HttpHost("3"))), + randomAlphaOfLength(5), randomAlphaOfLength(5), randomRoles()), // This node isn't in the original list so it isn't added: - new Node(new HttpHost("4"), emptyList(), randomAlphaOfLength(5), randomRoles()), + new Node(new HttpHost("4"), emptySet(), randomAlphaOfLength(5), + randomAlphaOfLength(5), randomRoles()), }); ESClientYamlSuiteTestCase.attachSniffedMetadataOnClient(client, originalNodes, nodesWithMetadata); verify(client).setNodes(new Node[] { @@ -139,12 +145,12 @@ public void testAttachSniffedMetadataOnClientNotEnoughNodes() { }; List nodesWithMetadata = Arrays.asList(new Node[] { // This node matches exactly: - new Node(new HttpHost("1"), emptyList(), "v", new Node.Roles(true, true, true)), + new Node(new HttpHost("1"), emptySet(), "n", "v", new Node.Roles(true, true, true)), }); IllegalStateException e = expectThrows(IllegalStateException.class, () -> ESClientYamlSuiteTestCase.attachSniffedMetadataOnClient(client, originalNodes, nodesWithMetadata)); assertEquals(e.getMessage(), "Didn't sniff metadata for all nodes. Wanted metadata for " - + "[http://1, http://2] but got [[host=http://1, bound=[], version=v, roles=mdi]]"); + + "[http://1, http://2] but got [[host=http://1, bound=[], name=n, version=v, roles=mdi]]"); verify(client, never()).setNodes(any(Node[].class)); } diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java index 588f5d17057b7..24731d2d52d5a 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/section/DoSectionTests.java @@ -535,7 +535,7 @@ public void testNodeSelector() throws IOException { } private Node nodeWithVersion(String version) { - return new Node(new HttpHost("dummy"), null, version, null); + return new Node(new HttpHost("dummy"), null, null, version, null); } private void assertJsonEquals(Map actual, String expected) throws IOException { From 3d4535b510bdf1e12cd76f7fb90d13d271308438 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 28 Mar 2018 17:26:51 -0400 Subject: [PATCH 25/27] Cleanups --- .../java/org/elasticsearch/client/Node.java | 9 ++-- .../elasticsearch/client/NodeSelector.java | 2 +- .../org/elasticsearch/client/RestClient.java | 53 +++++++++---------- .../elasticsearch/client/RestClientTests.java | 45 ++++++++-------- .../sniff/ElasticsearchHostsSnifferTests.java | 6 +-- 5 files changed, 55 insertions(+), 60 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/Node.java b/client/rest/src/main/java/org/elasticsearch/client/Node.java index 5345ba59bd24e..b26a0fa603c99 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/Node.java +++ b/client/rest/src/main/java/org/elasticsearch/client/Node.java @@ -82,8 +82,7 @@ public Node(HttpHost host) { /** * Make a copy of this {@link Node} but replacing its * {@link #getHost() host}. Use this when the sniffing implementation - * returns returns a {@link #getHost() host} that is not useful to the - * client. + * returns a {@link #getHost() host} that is not useful to the client. */ public Node withHost(HttpHost host) { /* @@ -192,19 +191,19 @@ public Roles(boolean masterEligible, boolean data, boolean ingest) { /** * The node could be elected master. */ - public boolean hasMasterEligible() { + public boolean isMasterEligible() { return masterEligible; } /** * The node stores data. */ - public boolean hasData() { + public boolean isData() { return data; } /** * The node runs ingest pipelines. */ - public boolean hasIngest() { + public boolean isIngest() { return ingest; } diff --git a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java index 4b20a97b2df3c..c95fd43395f86 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java +++ b/client/rest/src/main/java/org/elasticsearch/client/NodeSelector.java @@ -74,7 +74,7 @@ public List select(List nodes) { List subset = new ArrayList<>(nodes.size()); for (Node node : nodes) { if (node.getRoles() == null) continue; - if (false == node.getRoles().hasMasterEligible() || node.getRoles().hasData()) { + if (false == node.getRoles().isMasterEligible() || node.getRoles().isData()) { subset.add(node); } } diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 7ea04a5b37e06..f9b40dccdf530 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -137,7 +137,6 @@ public static RestClientBuilder builder(Node... nodes) { /** * Replaces the nodes that the client communicates without providing any * metadata about any of the nodes. - * @see Node#Node(HttpHost) */ public void setHosts(HttpHost... hosts) { setNodes(hostsToNodes(hosts)); @@ -255,11 +254,11 @@ void performRequestAsyncNoCatch(String method, String endpoint, Map> hostTuple, final HttpRequestBase request, + private void performRequestAsync(final long startTime, final NodeTuple> hostTuple, final HttpRequestBase request, final Set ignoreErrorCodes, final HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory, final FailureTrackingResponseListener listener) { - final HttpHost host = hostTuple.nodes.next(); + final HttpHost host = hostTuple.nodes.next().getHost(); //we stream the request body if the entity allows for it final HttpAsyncRequestProducer requestProducer = HttpAsyncMethods.create(host, request); final HttpAsyncResponseConsumer asyncResponseConsumer = @@ -360,30 +359,15 @@ private void setHeaders(HttpRequest httpRequest, Header[] requestHeaders) { * that. If the retries fail this throws a {@link IOException}. * @throws IOException if no nodes are available */ - private NodeTuple> nextNode(NodeSelector nodeSelector) throws IOException { + private NodeTuple> nextNode(NodeSelector nodeSelector) throws IOException { NodeTuple> nodeTuple = this.nodeTuple; - List hosts = selectHosts(nodeTuple, blacklist, lastNodeIndex, System.nanoTime(), nodeSelector); + List hosts = selectHosts(nodeTuple, blacklist, lastNodeIndex, System.nanoTime(), nodeSelector); return new NodeTuple<>(hosts.iterator(), nodeTuple.authCache); } - static List selectHosts(NodeTuple> nodeTuple, + static List selectHosts(NodeTuple> nodeTuple, Map blacklist, AtomicInteger lastNodeIndex, long now, NodeSelector nodeSelector) throws IOException { - class DeadNodeAndRevival { - final Node node; - final long nanosUntilRevival; - - DeadNodeAndRevival(Node node, long nanosUntilRevival) { - this.node = node; - this.nanosUntilRevival = nanosUntilRevival; - } - - @Override - public String toString() { - return node.toString(); - } - } - /* * Sort the nodes into living and dead lists. */ @@ -415,11 +399,7 @@ public String toString() { Collections.rotate(livingNodes, lastNodeIndex.getAndIncrement()); List selectedLivingNodes = nodeSelector.select(livingNodes); if (false == selectedLivingNodes.isEmpty()) { - List hosts = new ArrayList<>(selectedLivingNodes.size()); - for (Node node : selectedLivingNodes) { - hosts.add(node.getHost()); - } - return hosts; + return selectedLivingNodes; } } @@ -448,7 +428,7 @@ public int compare(DeadNodeAndRevival lhs, DeadNodeAndRevival rhs) { } selectedDeadNodes = nodeSelector.select(selectedDeadNodes); if (false == selectedDeadNodes.isEmpty()) { - return singletonList(selectedDeadNodes.get(0).getHost()); + return singletonList(selectedDeadNodes.get(0)); } } throw new IOException("NodeSelector [" + nodeSelector + "] rejected all nodes, " @@ -633,4 +613,23 @@ static class NodeTuple { this.authCache = authCache; } } + + /** + * Contains a reference to a blacklisted node and the time until it is + * revived. We use this so we can do a single pass over the blacklist. + */ + private static class DeadNodeAndRevival { + final Node node; + final long nanosUntilRevival; + + DeadNodeAndRevival(Node node, long nanosUntilRevival) { + this.node = node; + this.nanosUntilRevival = nanosUntilRevival; + } + + @Override + public String toString() { + return node.toString(); + } + } } diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java index 590c15e5a9362..cbc3292661cf0 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java @@ -158,13 +158,10 @@ public void testBuildUriLeavesPathUntouched() { public void testSelectHosts() throws IOException { int iterations = 1000; - HttpHost h1 = new HttpHost("1"); - HttpHost h2 = new HttpHost("2"); - HttpHost h3 = new HttpHost("3"); - List nodes = Arrays.asList( - new Node(h1, null, null, "1", null), - new Node(h2, null, null, "2", null), - new Node(h3, null, null, "3", null)); + Node n1 = new Node(new HttpHost("1"), null, null, "1", null); + Node n2 = new Node(new HttpHost("2"), null, null, "2", null); + Node n3 = new Node(new HttpHost("3"), null, null, "3", null); + List nodes = Arrays.asList(n1, n2, n3); NodeSelector not1 = new NodeSelector() { @Override @@ -201,33 +198,33 @@ public String toString() { long now = 0; // Normal case - List expectedHosts = Arrays.asList(h1, h2, h3); - assertEquals(expectedHosts, RestClient.selectHosts(nodeTuple, blacklist, + List expectedNodes = Arrays.asList(n1, n2, n3); + assertEquals(expectedNodes, RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, NodeSelector.ANY)); // Calling it again rotates the set of results for (int i = 0; i < iterations; i++) { - Collections.rotate(expectedHosts, 1); - assertEquals(expectedHosts, RestClient.selectHosts(nodeTuple, blacklist, + Collections.rotate(expectedNodes, 1); + assertEquals(expectedNodes, RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, NodeSelector.ANY)); } // Exclude some node lastNodeIndex.set(0); // h1 excluded - assertEquals(Arrays.asList(h2, h3), RestClient.selectHosts(nodeTuple, blacklist, + assertEquals(Arrays.asList(n2, n3), RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, not1)); // Calling it again rotates the set of results - assertEquals(Arrays.asList(h3, h2), RestClient.selectHosts(nodeTuple, blacklist, + assertEquals(Arrays.asList(n3, n2), RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, not1)); // And again, same - assertEquals(Arrays.asList(h2, h3), RestClient.selectHosts(nodeTuple, blacklist, + assertEquals(Arrays.asList(n2, n3), RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, not1)); /* * But this time it doesn't because the list being filtered changes * from (h1, h2, h3) to (h2, h3, h1) which both look the same when * you filter out h1. */ - assertEquals(Arrays.asList(h2, h3), RestClient.selectHosts(nodeTuple, blacklist, + assertEquals(Arrays.asList(n2, n3), RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, not1)); /* @@ -249,18 +246,18 @@ public String toString() { * Mark all nodes as dead and look up at a time *after* the * revival time. This should return all nodes. */ - blacklist.put(h1, new DeadHostState(1, 1)); - blacklist.put(h2, new DeadHostState(1, 2)); - blacklist.put(h3, new DeadHostState(1, 3)); + blacklist.put(n1.getHost(), new DeadHostState(1, 1)); + blacklist.put(n2.getHost(), new DeadHostState(1, 2)); + blacklist.put(n3.getHost(), new DeadHostState(1, 3)); lastNodeIndex.set(0); now = 1000; - expectedHosts = Arrays.asList(h1, h2, h3); - assertEquals(expectedHosts, RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, + expectedNodes = Arrays.asList(n1, n2, n3); + assertEquals(expectedNodes, RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, NodeSelector.ANY)); // Calling it again rotates the set of results for (int i = 0; i < iterations; i++) { - Collections.rotate(expectedHosts, 1); - assertEquals(expectedHosts, RestClient.selectHosts(nodeTuple, blacklist, + Collections.rotate(expectedNodes, 1); + assertEquals(expectedNodes, RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, NodeSelector.ANY)); } @@ -269,7 +266,7 @@ public String toString() { * Only the node closest to revival should come back. */ now = 0; - assertEquals(singletonList(h1), RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, + assertEquals(singletonList(n1), RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, NodeSelector.ANY)); /* @@ -277,7 +274,7 @@ public String toString() { * *and* a node selector that removes the node that is closest * to being revived. The second closest node should come back. */ - assertEquals(singletonList(h2), RestClient.selectHosts(nodeTuple, blacklist, + assertEquals(singletonList(n2), RestClient.selectHosts(nodeTuple, blacklist, lastNodeIndex, now, not1)); /* diff --git a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java index 0f3501b55311f..8ce55f4256530 100644 --- a/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java +++ b/client/sniffer/src/test/java/org/elasticsearch/client/sniff/ElasticsearchHostsSnifferTests.java @@ -244,13 +244,13 @@ private static SniffResponse buildSniffResponse(ElasticsearchHostsSniffer.Scheme Collections.shuffle(roles, getRandom()); generator.writeArrayFieldStart("roles"); for (String role : roles) { - if ("master".equals(role) && node.getRoles().hasMasterEligible()) { + if ("master".equals(role) && node.getRoles().isMasterEligible()) { generator.writeString("master"); } - if ("data".equals(role) && node.getRoles().hasData()) { + if ("data".equals(role) && node.getRoles().isData()) { generator.writeString("data"); } - if ("ingest".equals(role) && node.getRoles().hasIngest()) { + if ("ingest".equals(role) && node.getRoles().isIngest()) { generator.writeString("ingest"); } } From 87cef5326e6834f6d4565f8ed75a04ae55e49943 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 28 Mar 2018 17:30:31 -0400 Subject: [PATCH 26/27] Remove unneeded --- .../src/main/java/org/elasticsearch/client/RestClient.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index f9b40dccdf530..210e0547764d5 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -90,12 +90,6 @@ public class RestClient extends AbstractRestClientActions implements Closeable { private static final Log logger = LogFactory.getLog(RestClient.class); - /** - * The maximum number of attempts that {@link #nextNode(NodeSelector)} makes - * before giving up and failing the request. - */ - private static final int MAX_NEXT_NODES_ATTEMPTS = 10; - private final CloseableHttpAsyncClient client; // We don't rely on default headers supported by HttpAsyncClient as those cannot be replaced. // These are package private for tests. From 88a3b5c42d5a6b6919b5e0c4bdc8861d45006a90 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 28 Mar 2018 18:12:13 -0400 Subject: [PATCH 27/27] More cleanup --- .../org/elasticsearch/client/RestClient.java | 15 +++-- .../client/NodeSelectorTests.java | 4 +- .../org/elasticsearch/client/NodeTests.java | 2 +- .../client/RestClientSingleHostTests.java | 57 ------------------ .../elasticsearch/client/RestClientTests.java | 59 +++++++++++++++++++ .../sniff/ElasticsearchHostsSniffer.java | 10 +++- .../rest/yaml/ESClientYamlSuiteTestCase.java | 16 ++--- .../yaml/ESClientYamlSuiteTestCaseTests.java | 10 ++-- 8 files changed, 91 insertions(+), 82 deletions(-) diff --git a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java index 210e0547764d5..0316fef30bf59 100644 --- a/client/rest/src/main/java/org/elasticsearch/client/RestClient.java +++ b/client/rest/src/main/java/org/elasticsearch/client/RestClient.java @@ -123,6 +123,9 @@ public static RestClientBuilder builder(HttpHost... hosts) { /** * Returns a new {@link RestClientBuilder} to help with {@link RestClient} creation. * Creates a new builder instance and sets the nodes that the client will send requests to. + *

+ * Prefer this to {@link #builder(Node...)} if you have metadata up front about the nodes. + * If y ou don't either one is fine. */ public static RestClientBuilder builder(Node... nodes) { return new RestClientBuilder(nodes); @@ -172,8 +175,8 @@ public synchronized void setNodes(Node... nodes) { /** * Copy of the list of nodes that the client knows about. */ - public Node[] getNodes() { // TODO is it ok to expose this? It feels excessive but we do use it in tests. - return nodeTuple.nodes.toArray(new Node[0]); + public List getNodes() { + return nodeTuple.nodes; } @Override @@ -384,10 +387,10 @@ static List selectHosts(NodeTuple> nodeTuple, if (false == livingNodes.isEmpty()) { /* * Normal state: there is at least one living node. Rotate the - * list so subsequent requests to will see prefer the nodes in - * a different order then run them through the NodeSelector so - * it can have its say in which nodes are ok and their ordering. - * If the selector is ok with any over the living nodes then use + * list so subsequent requests to will prefer the nodes in a + * different order then run them through the NodeSelector so it + * can have its say in which nodes are ok and their ordering. If + * the selector is ok with any over the living nodes then use * them for the request. */ Collections.rotate(livingNodes, lastNodeIndex.getAndIncrement()); diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java index a8f5c70c33fe8..e8aa7a175be8b 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeSelectorTests.java @@ -42,9 +42,9 @@ public void testAny() { public void testNotMasterOnly() { Node masterOnly = dummyNode(true, false, randomBoolean()); Node masterAndData = dummyNode(true, true, randomBoolean()); - Node client = dummyNode(false, false, randomBoolean()); + Node coordinatingOnly = dummyNode(false, false, randomBoolean()); Node data = dummyNode(false, true, randomBoolean()); - List nodes = Arrays.asList(masterOnly, masterAndData, client, data); + List nodes = Arrays.asList(masterOnly, masterAndData, coordinatingOnly, data); Collections.shuffle(nodes, getRandom()); List expected = new ArrayList<>(nodes); expected.remove(masterOnly); diff --git a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java index 5ea1eb8268a0b..989861df50293 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/NodeTests.java @@ -40,7 +40,7 @@ public void testWithHost() { randomAsciiAlphanumOfLength(5), randomAsciiAlphanumOfLength(5), new Roles(randomBoolean(), randomBoolean(), randomBoolean())); - // Host is in nthe bound hosts list + // Host is in the bound hosts list assertEquals(h2, n.withHost(h2).getHost()); assertEquals(n.getBoundHosts(), n.withHost(h2).getBoundHosts()); diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java index 2147acf65810c..f33fe26f34e06 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostTests.java @@ -197,63 +197,6 @@ public void testInternalHttpRequest() throws Exception { } } - public void testSetHostsFailures() throws IOException { - try { - restClient.setHosts((HttpHost[]) null); - fail("setHosts should have failed"); - } catch (IllegalArgumentException e) { - assertEquals("hosts must not be null or empty", e.getMessage()); - } - try { - restClient.setHosts(); - fail("setHosts should have failed"); - } catch (IllegalArgumentException e) { - assertEquals("hosts must not be null or empty", e.getMessage()); - } - try { - restClient.setHosts((HttpHost) null); - fail("setHosts should have failed"); - } catch (IllegalArgumentException e) { - assertEquals("host cannot be null", e.getMessage()); - } - try { - restClient.setHosts(new HttpHost("localhost", 9200), null, new HttpHost("localhost", 9201)); - fail("setHosts should have failed"); - } catch (IllegalArgumentException e) { - assertEquals("host cannot be null", e.getMessage()); - } - } - - public void testSetNodesFailures() throws IOException { - try { - restClient.setNodes((Node[]) null); - fail("setNodes should have failed"); - } catch (IllegalArgumentException e) { - assertEquals("nodes must not be null or empty", e.getMessage()); - } - try { - restClient.setNodes(); - fail("setNodes should have failed"); - } catch (IllegalArgumentException e) { - assertEquals("nodes must not be null or empty", e.getMessage()); - } - try { - restClient.setNodes((Node) null); - fail("setNodes should have failed"); - } catch (IllegalArgumentException e) { - assertEquals("node cannot be null", e.getMessage()); - } - try { - restClient.setNodes( - new Node(new HttpHost("localhost", 9200)), - null, - new Node(new HttpHost("localhost", 9201))); - fail("setNodes should have failed"); - } catch (IllegalArgumentException e) { - assertEquals("node cannot be null", e.getMessage()); - } - } - /** * End to end test for ok status codes */ diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java index cbc3292661cf0..cca16162a7de6 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientTests.java @@ -295,6 +295,65 @@ public String toString() { } } + public void testSetHostsFailures() throws IOException { + RestClient restClient = createRestClient(); + try { + restClient.setHosts((HttpHost[]) null); + fail("setHosts should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("hosts must not be null or empty", e.getMessage()); + } + try { + restClient.setHosts(); + fail("setHosts should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("hosts must not be null or empty", e.getMessage()); + } + try { + restClient.setHosts((HttpHost) null); + fail("setHosts should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("host cannot be null", e.getMessage()); + } + try { + restClient.setHosts(new HttpHost("localhost", 9200), null, new HttpHost("localhost", 9201)); + fail("setHosts should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("host cannot be null", e.getMessage()); + } + } + + public void testSetNodesFailures() throws IOException { + RestClient restClient = createRestClient(); + try { + restClient.setNodes((Node[]) null); + fail("setNodes should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("nodes must not be null or empty", e.getMessage()); + } + try { + restClient.setNodes(); + fail("setNodes should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("nodes must not be null or empty", e.getMessage()); + } + try { + restClient.setNodes((Node) null); + fail("setNodes should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("node cannot be null", e.getMessage()); + } + try { + restClient.setNodes( + new Node(new HttpHost("localhost", 9200)), + null, + new Node(new HttpHost("localhost", 9201))); + fail("setNodes should have failed"); + } catch (IllegalArgumentException e) { + assertEquals("node cannot be null", e.getMessage()); + } + } + private static RestClient createRestClient() { Node[] hosts = new Node[] { new Node(new HttpHost("localhost", 9200)) diff --git a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java index 4ee57692feb58..5f91191f731bd 100644 --- a/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java +++ b/client/sniffer/src/main/java/org/elasticsearch/client/sniff/ElasticsearchHostsSniffer.java @@ -113,7 +113,10 @@ static List readHosts(HttpEntity entity, Scheme scheme, JsonFactory jsonFa JsonToken token = parser.nextToken(); assert token == JsonToken.START_OBJECT; String nodeId = parser.getCurrentName(); - readHost(nodeId, parser, scheme, nodes); + Node node = readHost(nodeId, parser, scheme); + if (node != null) { + nodes.add(node); + } } } else { parser.skipChildren(); @@ -124,7 +127,7 @@ static List readHosts(HttpEntity entity, Scheme scheme, JsonFactory jsonFa } } - private static void readHost(String nodeId, JsonParser parser, Scheme scheme, List nodes) throws IOException { + private static Node readHost(String nodeId, JsonParser parser, Scheme scheme) throws IOException { HttpHost publishedHost = null; /* * We sniff the bound hosts so we can look up the node based on any @@ -212,6 +215,7 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Li //http section is not present if http is not enabled on the node, ignore such nodes if (publishedHost == null) { logger.debug("skipping node [" + nodeId + "] with http disabled"); + return null; } else { logger.trace("adding node [" + nodeId + "]"); if (version.startsWith("2.")) { @@ -226,7 +230,7 @@ private static void readHost(String nodeId, JsonParser parser, Scheme scheme, Li } assert boundHosts.contains(publishedHost) : "[" + nodeId + "] doesn't make sense! publishedHost should be in boundHosts"; - nodes.add(new Node(publishedHost, boundHosts, name, version, new Roles(master, data, ingest))); + return new Node(publishedHost, boundHosts, name, version, new Roles(master, data, ingest)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index a72d25cdf0545..bc785fbc9748f 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -373,7 +373,7 @@ protected boolean randomizeContentType() { * */ private void sniffHostMetadata(RestClient client) throws IOException { - Node[] nodes = client.getNodes(); + List nodes = client.getNodes(); boolean allHaveRoles = true; for (Node node : nodes) { if (node.getRoles() == null) { @@ -390,16 +390,18 @@ private void sniffHostMetadata(RestClient client) throws IOException { ElasticsearchHostsSniffer.Scheme.valueOf(getProtocol().toUpperCase(Locale.ROOT)); /* * We don't want to change the list of nodes that the client communicates with - * because that'd just be rude. So instead we replace the nodes with nodes the - * that + * because that'd just be rude. So instead we replace the nodes find the nodes + * returned by the sniffer that correspond with the nodes already the client + * and set the nodes to them. That *shouldn't* change the nodes that the client + * communicates with. */ ElasticsearchHostsSniffer sniffer = new ElasticsearchHostsSniffer( adminClient(), ElasticsearchHostsSniffer.DEFAULT_SNIFF_REQUEST_TIMEOUT, scheme); attachSniffedMetadataOnClient(client, nodes, sniffer.sniffHosts()); } - static void attachSniffedMetadataOnClient(RestClient client, Node[] originalNodes, List nodesWithMetadata) { - Set originalHosts = Arrays.stream(originalNodes) + static void attachSniffedMetadataOnClient(RestClient client, List originalNodes, List nodesWithMetadata) { + Set originalHosts = originalNodes.stream() .map(Node::getHost) .collect(Collectors.toSet()); List sniffed = new ArrayList<>(); @@ -422,9 +424,9 @@ static void attachSniffedMetadataOnClient(RestClient client, Node[] originalNode } } } - int missing = originalNodes.length - sniffed.size(); + int missing = originalNodes.size() - sniffed.size(); if (missing > 0) { - List hosts = Arrays.stream(originalNodes) + List hosts = originalNodes.stream() .map(Node::getHost) .collect(Collectors.toList()); throw new IllegalStateException("Didn't sniff metadata for all nodes. Wanted metadata for " diff --git a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java index cd01e355f5885..b40b0048a538a 100644 --- a/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java +++ b/test/framework/src/test/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCaseTests.java @@ -106,11 +106,10 @@ private static void assertSingleFile(Set files, String dirName, String fil public void testAttachSniffedMetadataOnClientOk() { RestClient client = mock(RestClient.class); - Node[] originalNodes = new Node[] { + List originalNodes = Arrays.asList( new Node(new HttpHost("1")), new Node(new HttpHost("2")), - new Node(new HttpHost("3")), - }; + new Node(new HttpHost("3"))); List nodesWithMetadata = Arrays.asList(new Node[] { // This node matches exactly: new Node(new HttpHost("1"), emptySet(), randomAlphaOfLength(5), @@ -139,10 +138,9 @@ public void testAttachSniffedMetadataOnClientOk() { public void testAttachSniffedMetadataOnClientNotEnoughNodes() { // Try a version of the call that should fail because it doesn't have all the results RestClient client = mock(RestClient.class); - Node[] originalNodes = new Node[] { + List originalNodes = Arrays.asList( new Node(new HttpHost("1")), - new Node(new HttpHost("2")), - }; + new Node(new HttpHost("2"))); List nodesWithMetadata = Arrays.asList(new Node[] { // This node matches exactly: new Node(new HttpHost("1"), emptySet(), "n", "v", new Node.Roles(true, true, true)),