From 9a3f8ad67365d837cf3b192878b0b0444d0b423e Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 26 Jan 2018 11:41:14 -0800 Subject: [PATCH 1/4] Add settings to control the max size and count of warning headers in responses Add a dynamic persistent cluster level setting "http.max_warning_header_count" to control the maximum number of warning headers in client HTTP responses. Defaults to unbounded. Add a dynamic persistent cluster level setting "http.max_warning_header_size" to control the maximum total size of warning headers in client HTTP responses. Defaults to unbounded. Closes #28301 --- docs/reference/modules/cluster/misc.asciidoc | 21 +++++ .../common/settings/ClusterSettings.java | 2 + .../common/util/concurrent/ThreadContext.java | 84 ++++++++++++++++++- .../http/HttpTransportSettings.java | 5 +- .../java/org/elasticsearch/node/Node.java | 8 ++ .../logging/DeprecationLoggerTests.java | 55 ++++++++++++ 6 files changed, 171 insertions(+), 4 deletions(-) diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc index 3963312c0f4ea..855a240f99855 100644 --- a/docs/reference/modules/cluster/misc.asciidoc +++ b/docs/reference/modules/cluster/misc.asciidoc @@ -56,3 +56,24 @@ PUT /_cluster/settings } ------------------------------- // CONSOLE + +[[cluster-warning-headers]] +==== Warning Headers +For every distinct warning, elastic cluster will return a new warning header in the client HTTP response. +Sometimes the amount of returned warning headers can be too large and exceed client configuration settings. +These dynamic settings allow to set the maximum count and size of warning headers in client http responses. +Once the maximum count or size is reached, any extra warning will not produce an additional warning header. +The default value for `http.max_warning_header_count` is unbounded. +The default value for `http.max_warning_header_size` is unbounded. + +[source,js] +------------------------------- +PUT /_cluster/settings +{ + "persistent" : { + "http.max_warning_header_count" : 62, + "http.max_warning_header_size" : "7Kb" + } +} +------------------------------- +// CONSOLE \ No newline at end of file diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index 804340d63ed11..a752bb7c477fa 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -245,6 +245,8 @@ public void apply(Settings value, Settings current, Settings previous) { HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH, HttpTransportSettings.SETTING_HTTP_MAX_CHUNK_SIZE, HttpTransportSettings.SETTING_HTTP_MAX_HEADER_SIZE, + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT, + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE, HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH, HttpTransportSettings.SETTING_HTTP_READ_TIMEOUT, HttpTransportSettings.SETTING_HTTP_RESET_COOKIES, diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java index 8f950c5434bd7..439c5c7042684 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java @@ -23,10 +23,16 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.http.HttpTransportSettings; + +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT; +import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE; import java.io.Closeable; import java.io.IOException; @@ -39,13 +45,14 @@ import java.util.Set; import java.util.concurrent.CancellationException; import java.util.concurrent.ExecutionException; -import java.util.concurrent.FutureTask; import java.util.concurrent.RunnableFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.nio.charset.StandardCharsets; + /** * A ThreadContext is a map of string headers and a transient map of keyed objects that are associated with @@ -81,6 +88,8 @@ public final class ThreadContext implements Closeable, Writeable { private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct(); private final Map defaultHeader; private final ContextThreadLocal threadLocal; + private static volatile int maxWrnHeaderCount; + private static volatile long maxWrnHeaderSize; /** * Creates a new ThreadContext instance @@ -98,6 +107,8 @@ public ThreadContext(Settings settings) { this.defaultHeader = Collections.unmodifiableMap(defaultHeader); } threadLocal = new ContextThreadLocal(); + maxWrnHeaderCount = SETTING_HTTP_MAX_WARNING_HEADER_COUNT.get(settings); + maxWrnHeaderSize = SETTING_HTTP_MAX_WARNING_HEADER_SIZE.get(settings).getBytes(); } @Override @@ -105,6 +116,14 @@ public void close() throws IOException { threadLocal.close(); } + public static void setMaxWarningHeaderCount(int newMaxWrnHeaderCount){ + maxWrnHeaderCount = newMaxWrnHeaderCount; + } + + public static void setMaxWarningHeaderSize(ByteSizeValue newMaxWarningHeaderSize){ + maxWrnHeaderSize = newMaxWarningHeaderSize.getBytes(); + } + /** * Removes the current context and resets a default context. The removed context can be * restored when closing the returned {@link StoredContext} @@ -359,7 +378,8 @@ private static final class ThreadContextStruct { private final Map transientHeaders; private final Map> responseHeaders; private final boolean isSystemContext; - + private long wrnHeadersSize; //saving current warning headers' size not to recalculate the size with every new warning header + private boolean isWrnLmtReached; private ThreadContextStruct(StreamInput in) throws IOException { final int numRequest = in.readVInt(); Map requestHeaders = numRequest == 0 ? Collections.emptyMap() : new HashMap<>(numRequest); @@ -371,6 +391,8 @@ private ThreadContextStruct(StreamInput in) throws IOException { this.responseHeaders = in.readMapOfLists(StreamInput::readString, StreamInput::readString); this.transientHeaders = Collections.emptyMap(); isSystemContext = false; // we never serialize this it's a transient flag + wrnHeadersSize = 0L; + isWrnLmtReached = false; } private ThreadContextStruct setSystemContext() { @@ -387,6 +409,20 @@ private ThreadContextStruct(Map requestHeaders, this.responseHeaders = responseHeaders; this.transientHeaders = transientHeaders; this.isSystemContext = isSystemContext; + this.wrnHeadersSize = 0L; + isWrnLmtReached = false; + } + + private ThreadContextStruct(Map requestHeaders, + Map> responseHeaders, + Map transientHeaders, boolean isSystemContext, + long wrnHeadersSize, boolean isWrnLmtReached) { + this.requestHeaders = requestHeaders; + this.responseHeaders = responseHeaders; + this.transientHeaders = transientHeaders; + this.isSystemContext = isSystemContext; + this.wrnHeadersSize = wrnHeadersSize; + this.isWrnLmtReached = isWrnLmtReached; } /** @@ -442,6 +478,19 @@ private ThreadContextStruct putResponseHeaders(Map> headers private ThreadContextStruct putResponse(final String key, final String value, final Function uniqueValue) { assert value != null; + long curWrnHeaderSize = 0; + //check if we can add another warning header (max count or size within limits) + if (key.equals("Warning")) { + if (isWrnLmtReached) return this; //can't add warning headers - limit reached + if (maxWrnHeaderCount != -1) { //if count is NOT unbounded, check its limits + int wrnHeaderCount = this.responseHeaders.containsKey("Warning") ? this.responseHeaders.get("Warning").size() : 0; + if (wrnHeaderCount >= maxWrnHeaderCount) return addWrnLmtReachedHeader(); + } + if (maxWrnHeaderSize != -1) { //if size is NOT unbounded, check its limits + curWrnHeaderSize = "Warning".getBytes(StandardCharsets.UTF_8).length + value.getBytes(StandardCharsets.UTF_8).length; + if ((wrnHeadersSize + curWrnHeaderSize) > maxWrnHeaderSize) return addWrnLmtReachedHeader(); + } + } final Map> newResponseHeaders = new HashMap<>(this.responseHeaders); final List existingValues = newResponseHeaders.get(key); @@ -460,8 +509,37 @@ private ThreadContextStruct putResponse(final String key, final String value, fi } else { newResponseHeaders.put(key, Collections.singletonList(value)); } + return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, + isSystemContext, wrnHeadersSize + curWrnHeaderSize, isWrnLmtReached); + } - return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext); + //replace last warning header(s) with "headers limit reached" warning + //respecting limitations on headers size if it is set by user + private ThreadContextStruct addWrnLmtReachedHeader(){ + if ((maxWrnHeaderSize == 0) || (maxWrnHeaderCount ==0)) //can't even add "headers limit reached" warning + return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, + isSystemContext, wrnHeadersSize, true); + final Map> newResponseHeaders = new HashMap<>(this.responseHeaders); + final List wrns = new ArrayList<>(newResponseHeaders.get("Warning")); + final String lastWrnMessage = DeprecationLogger.formatWarning( + "There were more warnings, but they were dropped as [" + + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey() + "] or [" + + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey() + "] were reached!"); + + if (maxWrnHeaderSize > 0) { + final long wrnSize = "Warning".getBytes(StandardCharsets.UTF_8).length; + wrnHeadersSize = wrnHeadersSize + wrnSize + lastWrnMessage.getBytes(StandardCharsets.UTF_8).length; + do { + String wrn = wrns.remove(wrns.size() - 1); + wrnHeadersSize = wrnHeadersSize - wrnSize - wrn.getBytes(StandardCharsets.UTF_8).length; + } while(wrnHeadersSize > maxWrnHeaderSize); + } else { //we don't care about size as it is unbounded + wrns.remove(wrns.size() - 1); + } + wrns.add(lastWrnMessage); + newResponseHeaders.put("Warning", Collections.unmodifiableList(wrns)); + return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, + isSystemContext, wrnHeadersSize, true); } private ThreadContextStruct putTransient(String key, Object value) { diff --git a/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java index 315fa5b038bfd..0767ecc6efc45 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java +++ b/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java @@ -29,7 +29,6 @@ import org.elasticsearch.common.unit.TimeValue; import java.util.List; -import java.util.concurrent.TimeUnit; import java.util.function.Function; import static java.util.Collections.emptyList; @@ -88,6 +87,10 @@ public final class HttpTransportSettings { Setting.byteSizeSetting("http.max_chunk_size", new ByteSizeValue(8, ByteSizeUnit.KB), Property.NodeScope); public static final Setting SETTING_HTTP_MAX_HEADER_SIZE = Setting.byteSizeSetting("http.max_header_size", new ByteSizeValue(8, ByteSizeUnit.KB), Property.NodeScope); + public static final Setting SETTING_HTTP_MAX_WARNING_HEADER_COUNT = + Setting.intSetting("http.max_warning_header_count", -1, -1, Setting.Property.Dynamic, Property.NodeScope); + public static final Setting SETTING_HTTP_MAX_WARNING_HEADER_SIZE = + Setting.byteSizeSetting("http.max_warning_header_size", new ByteSizeValue(-1), Setting.Property.Dynamic, Property.NodeScope); public static final Setting SETTING_HTTP_MAX_INITIAL_LINE_LENGTH = Setting.byteSizeSetting("http.max_initial_line_length", new ByteSizeValue(4, ByteSizeUnit.KB), Property.NodeScope); // don't reset cookies by default, since I don't think we really need to diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index 081c588525ed9..bb44d94d0b81e 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -20,6 +20,7 @@ package org.elasticsearch.node; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.ThreadContext; import org.apache.lucene.util.Constants; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.SetOnce; @@ -93,6 +94,7 @@ import org.elasticsearch.gateway.GatewayService; import org.elasticsearch.gateway.MetaStateService; import org.elasticsearch.http.HttpServerTransport; +import org.elasticsearch.http.HttpTransportSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.indices.IndicesService; @@ -351,6 +353,12 @@ protected Node(final Environment environment, Collection listener::onNewInfo); final UsageService usageService = new UsageService(settings); + + clusterService.getClusterSettings().addSettingsUpdateConsumer(HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT, + org.elasticsearch.common.util.concurrent.ThreadContext::setMaxWarningHeaderCount); + clusterService.getClusterSettings().addSettingsUpdateConsumer(HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE, + org.elasticsearch.common.util.concurrent.ThreadContext::setMaxWarningHeaderSize); + ModulesBuilder modules = new ModulesBuilder(); // plugin modules must be added here, before others or we can get crazy injection errors... for (Module pluginModule : pluginsService.createGuiceModules()) { diff --git a/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java b/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java index fdb530749e105..3ec0bb3a6e050 100644 --- a/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java +++ b/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.IntStream; +import java.nio.charset.StandardCharsets; import static org.elasticsearch.common.logging.DeprecationLogger.WARNING_HEADER_PATTERN; import static org.elasticsearch.test.hamcrest.RegexMatcher.matches; @@ -246,6 +247,60 @@ public void testEncode() { assertThat(DeprecationLogger.encode(s), IsSame.sameInstance(s)); } + + public void testWarningHeaderCountSetting() throws IOException{ + // Test that the number of warning headers don't exceed 'http.max_warning_header_count' + final int maxWarningHeaderCount = 2; + Settings settings = Settings.builder() + .put("http.max_warning_header_count", maxWarningHeaderCount) + .build(); + try (ThreadContext threadContext = new ThreadContext(settings)) { + final Set threadContexts = Collections.singleton(threadContext); + // try to log three warning messages + logger.deprecated(threadContexts, "A simple message 1"); + logger.deprecated(threadContexts, "A simple message 2"); + logger.deprecated(threadContexts, "A simple message 3"); + final Map> responseHeaders = threadContext.getResponseHeaders(); + final List responses = responseHeaders.get("Warning"); + + assertEquals(maxWarningHeaderCount, responses.size()); + assertThat(responses.get(0), warningValueMatcher); + assertThat(responses.get(0), containsString("\"A simple message 1")); + assertThat(responses.get(1), warningValueMatcher); + assertThat(responses.get(1), containsString("\"There were more warnings, but they were dropped as ")); + } + } + + public void testWarningHeaderSizeSetting() throws IOException{ + // Test that the size of warning headers don't exceed 'http.max_warning_header_size' + Settings settings = Settings.builder() + .put("http.max_warning_header_size", "1Kb") + .build(); + + byte [] arr = new byte[300]; + String message1 = new String(arr, StandardCharsets.UTF_8) + "1"; + String message2 = new String(arr, StandardCharsets.UTF_8) + "2"; + String message3 = new String(arr, StandardCharsets.UTF_8) + "3"; + + try (ThreadContext threadContext = new ThreadContext(settings)) { + final Set threadContexts = Collections.singleton(threadContext); + // try to log three warning messages + logger.deprecated(threadContexts, message1); + logger.deprecated(threadContexts, message2); + logger.deprecated(threadContexts, message3); + final Map> responseHeaders = threadContext.getResponseHeaders(); + final List responses = responseHeaders.get("Warning"); + + long warningHeadersSize = 0L; + for (String response : responses){ + warningHeadersSize += "Warning".getBytes(StandardCharsets.UTF_8).length + + response.getBytes(StandardCharsets.UTF_8).length; + } + // assert that the size of all warning headers is less or equal to 1Kb + assertTrue(warningHeadersSize <= 1024); + } + } + private String range(int lowerInclusive, int upperInclusive) { return IntStream .range(lowerInclusive, upperInclusive + 1) From 0fa44fb1235f917ee5dcde4cf8728afffa87351c Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Tue, 20 Mar 2018 15:47:17 -0700 Subject: [PATCH 2/4] Control max size and count of warning headers Add a dynamic persistent cluster level setting "http.max_warning_header_count" to control the maximum number of warning headers in client HTTP responses. Defaults to unbounded. Add a dynamic persistent cluster level setting "http.max_warning_header_size" to control the maximum total size of warning headers in client HTTP responses. Defaults to unbounded. Once any of these limits is exceeded this will be logged in the main ES log, and any more warning headers for this response will be ignored. Closes #28301 --- docs/reference/modules/cluster/misc.asciidoc | 21 ---- docs/reference/modules/http.asciidoc | 14 ++- .../common/util/concurrent/ThreadContext.java | 102 ++++++++---------- .../logging/DeprecationLoggerTests.java | 2 +- 4 files changed, 56 insertions(+), 83 deletions(-) diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc index 855a240f99855..05e5b4ab3e19d 100644 --- a/docs/reference/modules/cluster/misc.asciidoc +++ b/docs/reference/modules/cluster/misc.asciidoc @@ -55,25 +55,4 @@ PUT /_cluster/settings } } ------------------------------- -// CONSOLE - -[[cluster-warning-headers]] -==== Warning Headers -For every distinct warning, elastic cluster will return a new warning header in the client HTTP response. -Sometimes the amount of returned warning headers can be too large and exceed client configuration settings. -These dynamic settings allow to set the maximum count and size of warning headers in client http responses. -Once the maximum count or size is reached, any extra warning will not produce an additional warning header. -The default value for `http.max_warning_header_count` is unbounded. -The default value for `http.max_warning_header_size` is unbounded. - -[source,js] -------------------------------- -PUT /_cluster/settings -{ - "persistent" : { - "http.max_warning_header_count" : 62, - "http.max_warning_header_size" : "7Kb" - } -} -------------------------------- // CONSOLE \ No newline at end of file diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc index a83270ec2aace..f7c27497ff604 100644 --- a/docs/reference/modules/http.asciidoc +++ b/docs/reference/modules/http.asciidoc @@ -18,9 +18,11 @@ http://en.wikipedia.org/wiki/Chunked_transfer_encoding[HTTP chunking]. [float] === Settings -The settings in the table below can be configured for HTTP. Note that none of -them are dynamically updatable so for them to take effect they should be set in -`elasticsearch.yml`. +The settings in the table below can be configured for HTTP. Note that +almost all of them are not dynamically updatable. For them to take effect +they should be set in `elasticsearch.yml`. The only dynamically updatable +settings are `http.max_warning_header_count` and +`http.max_warning_header_size`. [cols="<,<",options="header",] |======================================================================= @@ -100,6 +102,12 @@ simple message will be returned. Defaults to `true` |`http.pipelining.max_events` |The maximum number of events to be queued up in memory before a HTTP connection is closed, defaults to `10000`. +|`http.max_warning_header_count` |The maximum number of warning headers in + client HTTP responses. Can be dynamically updated. Defaults to unbounded. + +|`http.max_warning_header_size` |The maximum total size of warning headers in +client HTTP responses. Can be dynamically updated. Defaults to unbounded. + |======================================================================= It also uses the common diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java index 439c5c7042684..f15689942255d 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java @@ -88,8 +88,8 @@ public final class ThreadContext implements Closeable, Writeable { private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct(); private final Map defaultHeader; private final ContextThreadLocal threadLocal; - private static volatile int maxWrnHeaderCount; - private static volatile long maxWrnHeaderSize; + private static volatile int maxWarningHeaderCount; + private static volatile long maxWarningHeaderSize; /** * Creates a new ThreadContext instance @@ -107,8 +107,8 @@ public ThreadContext(Settings settings) { this.defaultHeader = Collections.unmodifiableMap(defaultHeader); } threadLocal = new ContextThreadLocal(); - maxWrnHeaderCount = SETTING_HTTP_MAX_WARNING_HEADER_COUNT.get(settings); - maxWrnHeaderSize = SETTING_HTTP_MAX_WARNING_HEADER_SIZE.get(settings).getBytes(); + maxWarningHeaderCount = SETTING_HTTP_MAX_WARNING_HEADER_COUNT.get(settings); + maxWarningHeaderSize = SETTING_HTTP_MAX_WARNING_HEADER_SIZE.get(settings).getBytes(); } @Override @@ -116,12 +116,12 @@ public void close() throws IOException { threadLocal.close(); } - public static void setMaxWarningHeaderCount(int newMaxWrnHeaderCount){ - maxWrnHeaderCount = newMaxWrnHeaderCount; + public static void setMaxWarningHeaderCount(final int maxWarningHeaderCount) { + ThreadContext.maxWarningHeaderCount = maxWarningHeaderCount; } - public static void setMaxWarningHeaderSize(ByteSizeValue newMaxWarningHeaderSize){ - maxWrnHeaderSize = newMaxWarningHeaderSize.getBytes(); + public static void setMaxWarningHeaderSize(final ByteSizeValue maxWarningHeaderSize) { + ThreadContext.maxWarningHeaderSize = maxWarningHeaderSize.getBytes(); } /** @@ -378,8 +378,8 @@ private static final class ThreadContextStruct { private final Map transientHeaders; private final Map> responseHeaders; private final boolean isSystemContext; - private long wrnHeadersSize; //saving current warning headers' size not to recalculate the size with every new warning header - private boolean isWrnLmtReached; + private long warningHeadersSize; //saving current warning headers' size not to recalculate the size with every new warning header + private boolean isWarningLimitReached; private ThreadContextStruct(StreamInput in) throws IOException { final int numRequest = in.readVInt(); Map requestHeaders = numRequest == 0 ? Collections.emptyMap() : new HashMap<>(numRequest); @@ -391,8 +391,8 @@ private ThreadContextStruct(StreamInput in) throws IOException { this.responseHeaders = in.readMapOfLists(StreamInput::readString, StreamInput::readString); this.transientHeaders = Collections.emptyMap(); isSystemContext = false; // we never serialize this it's a transient flag - wrnHeadersSize = 0L; - isWrnLmtReached = false; + this.warningHeadersSize = 0L; + this.isWarningLimitReached = false; } private ThreadContextStruct setSystemContext() { @@ -409,20 +409,20 @@ private ThreadContextStruct(Map requestHeaders, this.responseHeaders = responseHeaders; this.transientHeaders = transientHeaders; this.isSystemContext = isSystemContext; - this.wrnHeadersSize = 0L; - isWrnLmtReached = false; + this.warningHeadersSize = 0L; + this.isWarningLimitReached = false; } private ThreadContextStruct(Map requestHeaders, Map> responseHeaders, Map transientHeaders, boolean isSystemContext, - long wrnHeadersSize, boolean isWrnLmtReached) { + long warningHeadersSize, boolean isWarningLimitReached) { this.requestHeaders = requestHeaders; this.responseHeaders = responseHeaders; this.transientHeaders = transientHeaders; this.isSystemContext = isSystemContext; - this.wrnHeadersSize = wrnHeadersSize; - this.isWrnLmtReached = isWrnLmtReached; + this.warningHeadersSize = warningHeadersSize; + this.isWarningLimitReached = isWarningLimitReached; } /** @@ -478,70 +478,56 @@ private ThreadContextStruct putResponseHeaders(Map> headers private ThreadContextStruct putResponse(final String key, final String value, final Function uniqueValue) { assert value != null; - long curWrnHeaderSize = 0; - //check if we can add another warning header (max count or size within limits) + long newWarningHeaderSize = warningHeadersSize; + //check if we can add another warning header - if max size within limits if (key.equals("Warning")) { - if (isWrnLmtReached) return this; //can't add warning headers - limit reached - if (maxWrnHeaderCount != -1) { //if count is NOT unbounded, check its limits - int wrnHeaderCount = this.responseHeaders.containsKey("Warning") ? this.responseHeaders.get("Warning").size() : 0; - if (wrnHeaderCount >= maxWrnHeaderCount) return addWrnLmtReachedHeader(); - } - if (maxWrnHeaderSize != -1) { //if size is NOT unbounded, check its limits - curWrnHeaderSize = "Warning".getBytes(StandardCharsets.UTF_8).length + value.getBytes(StandardCharsets.UTF_8).length; - if ((wrnHeadersSize + curWrnHeaderSize) > maxWrnHeaderSize) return addWrnLmtReachedHeader(); + if (isWarningLimitReached) return this; // can't add warning headers - limit reached + newWarningHeaderSize += "Warning".getBytes(StandardCharsets.UTF_8).length + value.getBytes(StandardCharsets.UTF_8).length; + //if size is NOT unbounded AND limit is exceeded + if ((maxWarningHeaderSize != -1) && (newWarningHeaderSize > maxWarningHeaderSize)) { + logWarningsLimitReached(); + return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, + isSystemContext, newWarningHeaderSize, true); } } final Map> newResponseHeaders = new HashMap<>(this.responseHeaders); final List existingValues = newResponseHeaders.get(key); - if (existingValues != null) { final Set existingUniqueValues = existingValues.stream().map(uniqueValue).collect(Collectors.toSet()); assert existingValues.size() == existingUniqueValues.size(); if (existingUniqueValues.contains(uniqueValue.apply(value))) { return this; } - final List newValues = new ArrayList<>(existingValues); newValues.add(value); - newResponseHeaders.put(key, Collections.unmodifiableList(newValues)); } else { newResponseHeaders.put(key, Collections.singletonList(value)); } - return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, - isSystemContext, wrnHeadersSize + curWrnHeaderSize, isWrnLmtReached); - } - //replace last warning header(s) with "headers limit reached" warning - //respecting limitations on headers size if it is set by user - private ThreadContextStruct addWrnLmtReachedHeader(){ - if ((maxWrnHeaderSize == 0) || (maxWrnHeaderCount ==0)) //can't even add "headers limit reached" warning - return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, - isSystemContext, wrnHeadersSize, true); - final Map> newResponseHeaders = new HashMap<>(this.responseHeaders); - final List wrns = new ArrayList<>(newResponseHeaders.get("Warning")); - final String lastWrnMessage = DeprecationLogger.formatWarning( - "There were more warnings, but they were dropped as [" + - HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey() + "] or [" + - HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey() + "] were reached!"); - - if (maxWrnHeaderSize > 0) { - final long wrnSize = "Warning".getBytes(StandardCharsets.UTF_8).length; - wrnHeadersSize = wrnHeadersSize + wrnSize + lastWrnMessage.getBytes(StandardCharsets.UTF_8).length; - do { - String wrn = wrns.remove(wrns.size() - 1); - wrnHeadersSize = wrnHeadersSize - wrnSize - wrn.getBytes(StandardCharsets.UTF_8).length; - } while(wrnHeadersSize > maxWrnHeaderSize); - } else { //we don't care about size as it is unbounded - wrns.remove(wrns.size() - 1); + //check if we can add another warning header - if max count within limits + if ((key.equals("Warning")) && (maxWarningHeaderCount != -1)) { //if count is NOT unbounded, check its limits + final int warningHeaderCount = newResponseHeaders.containsKey("Warning") ? newResponseHeaders.get("Warning").size() : 0; + if (warningHeaderCount > maxWarningHeaderCount) { + logWarningsLimitReached(); + return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, + isSystemContext, newWarningHeaderSize, true); + } } - wrns.add(lastWrnMessage); - newResponseHeaders.put("Warning", Collections.unmodifiableList(wrns)); return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, - isSystemContext, wrnHeadersSize, true); + isSystemContext, newWarningHeaderSize, isWarningLimitReached); + } + + + private void logWarningsLimitReached() { + final String message = "There were more warnings, but they were dropped as [" + + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey() + "] or [" + + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey() + "] were reached!"; + ESLoggerFactory.getLogger(ThreadContext.class).warn(message); } + private ThreadContextStruct putTransient(String key, Object value) { Map newTransient = new HashMap<>(this.transientHeaders); if (newTransient.putIfAbsent(key, value) != null) { diff --git a/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java b/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java index 3ec0bb3a6e050..490f7961a894d 100644 --- a/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java +++ b/server/src/test/java/org/elasticsearch/common/logging/DeprecationLoggerTests.java @@ -267,7 +267,7 @@ public void testWarningHeaderCountSetting() throws IOException{ assertThat(responses.get(0), warningValueMatcher); assertThat(responses.get(0), containsString("\"A simple message 1")); assertThat(responses.get(1), warningValueMatcher); - assertThat(responses.get(1), containsString("\"There were more warnings, but they were dropped as ")); + assertThat(responses.get(1), containsString("\"A simple message 2")); } } From 63dc6b85b694dc94c459673a0fda86febebe8b26 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Thu, 29 Mar 2018 11:09:39 -0700 Subject: [PATCH 3/4] Control max size and count of warning headers Add a static persistent cluster level setting "http.max_warning_header_count" to control the maximum number of warning headers in client HTTP responses. Defaults to unbounded. Add a static persistent cluster level setting "http.max_warning_header_size" to control the maximum total size of warning headers in client HTTP responses. Defaults to unbounded. Once any of these limits is exceeded this will be logged in the main ES log, and any more warning headers for this response will be ignored. --- docs/reference/modules/cluster/misc.asciidoc | 3 +-- docs/reference/modules/http.asciidoc | 12 +++++------ .../common/util/concurrent/ThreadContext.java | 21 +++++++------------ .../http/HttpTransportSettings.java | 4 ++-- .../java/org/elasticsearch/node/Node.java | 7 ------- 5 files changed, 15 insertions(+), 32 deletions(-) diff --git a/docs/reference/modules/cluster/misc.asciidoc b/docs/reference/modules/cluster/misc.asciidoc index 05e5b4ab3e19d..9238fdf9e08b3 100644 --- a/docs/reference/modules/cluster/misc.asciidoc +++ b/docs/reference/modules/cluster/misc.asciidoc @@ -54,5 +54,4 @@ PUT /_cluster/settings "logger.org.elasticsearch.indices.recovery": "DEBUG" } } -------------------------------- -// CONSOLE \ No newline at end of file +------------------------------- \ No newline at end of file diff --git a/docs/reference/modules/http.asciidoc b/docs/reference/modules/http.asciidoc index f7c27497ff604..ee9423c1e1c65 100644 --- a/docs/reference/modules/http.asciidoc +++ b/docs/reference/modules/http.asciidoc @@ -18,11 +18,9 @@ http://en.wikipedia.org/wiki/Chunked_transfer_encoding[HTTP chunking]. [float] === Settings -The settings in the table below can be configured for HTTP. Note that -almost all of them are not dynamically updatable. For them to take effect -they should be set in `elasticsearch.yml`. The only dynamically updatable -settings are `http.max_warning_header_count` and -`http.max_warning_header_size`. +The settings in the table below can be configured for HTTP. Note that none of +them are dynamically updatable so for them to take effect they should be set in +the Elasticsearch <>. [cols="<,<",options="header",] |======================================================================= @@ -103,10 +101,10 @@ simple message will be returned. Defaults to `true` |`http.pipelining.max_events` |The maximum number of events to be queued up in memory before a HTTP connection is closed, defaults to `10000`. |`http.max_warning_header_count` |The maximum number of warning headers in - client HTTP responses. Can be dynamically updated. Defaults to unbounded. + client HTTP responses, defaults to unbounded. |`http.max_warning_header_size` |The maximum total size of warning headers in -client HTTP responses. Can be dynamically updated. Defaults to unbounded. +client HTTP responses, defaults to unbounded. |======================================================================= diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java index f15689942255d..a3946058fe061 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java @@ -88,8 +88,8 @@ public final class ThreadContext implements Closeable, Writeable { private static final ThreadContextStruct DEFAULT_CONTEXT = new ThreadContextStruct(); private final Map defaultHeader; private final ContextThreadLocal threadLocal; - private static volatile int maxWarningHeaderCount; - private static volatile long maxWarningHeaderSize; + private final int maxWarningHeaderCount; + private final long maxWarningHeaderSize; /** * Creates a new ThreadContext instance @@ -107,8 +107,8 @@ public ThreadContext(Settings settings) { this.defaultHeader = Collections.unmodifiableMap(defaultHeader); } threadLocal = new ContextThreadLocal(); - maxWarningHeaderCount = SETTING_HTTP_MAX_WARNING_HEADER_COUNT.get(settings); - maxWarningHeaderSize = SETTING_HTTP_MAX_WARNING_HEADER_SIZE.get(settings).getBytes(); + this.maxWarningHeaderCount = SETTING_HTTP_MAX_WARNING_HEADER_COUNT.get(settings); + this.maxWarningHeaderSize = SETTING_HTTP_MAX_WARNING_HEADER_SIZE.get(settings).getBytes(); } @Override @@ -116,14 +116,6 @@ public void close() throws IOException { threadLocal.close(); } - public static void setMaxWarningHeaderCount(final int maxWarningHeaderCount) { - ThreadContext.maxWarningHeaderCount = maxWarningHeaderCount; - } - - public static void setMaxWarningHeaderSize(final ByteSizeValue maxWarningHeaderSize) { - ThreadContext.maxWarningHeaderSize = maxWarningHeaderSize.getBytes(); - } - /** * Removes the current context and resets a default context. The removed context can be * restored when closing the returned {@link StoredContext} @@ -301,7 +293,7 @@ public void addResponseHeader(final String key, final String value) { * @param uniqueValue the function that produces de-duplication values */ public void addResponseHeader(final String key, final String value, final Function uniqueValue) { - threadLocal.set(threadLocal.get().putResponse(key, value, uniqueValue)); + threadLocal.set(threadLocal.get().putResponse(key, value, uniqueValue, maxWarningHeaderCount, maxWarningHeaderSize)); } /** @@ -476,7 +468,8 @@ private ThreadContextStruct putResponseHeaders(Map> headers return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext); } - private ThreadContextStruct putResponse(final String key, final String value, final Function uniqueValue) { + private ThreadContextStruct putResponse(final String key, final String value, final Function uniqueValue, + final int maxWarningHeaderCount, final long maxWarningHeaderSize) { assert value != null; long newWarningHeaderSize = warningHeadersSize; //check if we can add another warning header - if max size within limits diff --git a/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java b/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java index 0767ecc6efc45..ce305872d222d 100644 --- a/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java +++ b/server/src/main/java/org/elasticsearch/http/HttpTransportSettings.java @@ -88,9 +88,9 @@ public final class HttpTransportSettings { public static final Setting SETTING_HTTP_MAX_HEADER_SIZE = Setting.byteSizeSetting("http.max_header_size", new ByteSizeValue(8, ByteSizeUnit.KB), Property.NodeScope); public static final Setting SETTING_HTTP_MAX_WARNING_HEADER_COUNT = - Setting.intSetting("http.max_warning_header_count", -1, -1, Setting.Property.Dynamic, Property.NodeScope); + Setting.intSetting("http.max_warning_header_count", -1, -1, Property.NodeScope); public static final Setting SETTING_HTTP_MAX_WARNING_HEADER_SIZE = - Setting.byteSizeSetting("http.max_warning_header_size", new ByteSizeValue(-1), Setting.Property.Dynamic, Property.NodeScope); + Setting.byteSizeSetting("http.max_warning_header_size", new ByteSizeValue(-1), Property.NodeScope); public static final Setting SETTING_HTTP_MAX_INITIAL_LINE_LENGTH = Setting.byteSizeSetting("http.max_initial_line_length", new ByteSizeValue(4, ByteSizeUnit.KB), Property.NodeScope); // don't reset cookies by default, since I don't think we really need to diff --git a/server/src/main/java/org/elasticsearch/node/Node.java b/server/src/main/java/org/elasticsearch/node/Node.java index ea07d3297a12b..9fa886af8bc68 100644 --- a/server/src/main/java/org/elasticsearch/node/Node.java +++ b/server/src/main/java/org/elasticsearch/node/Node.java @@ -20,7 +20,6 @@ package org.elasticsearch.node; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.ThreadContext; import org.apache.lucene.util.Constants; import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.SetOnce; @@ -353,12 +352,6 @@ protected Node(final Environment environment, Collection listener::onNewInfo); final UsageService usageService = new UsageService(settings); - - clusterService.getClusterSettings().addSettingsUpdateConsumer(HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT, - org.elasticsearch.common.util.concurrent.ThreadContext::setMaxWarningHeaderCount); - clusterService.getClusterSettings().addSettingsUpdateConsumer(HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE, - org.elasticsearch.common.util.concurrent.ThreadContext::setMaxWarningHeaderSize); - ModulesBuilder modules = new ModulesBuilder(); // plugin modules must be added here, before others or we can get crazy injection errors... for (Module pluginModule : pluginsService.createGuiceModules()) { From 4e2fc02ad2967d265d1654f10568d3f34da563e0 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Thu, 5 Apr 2018 16:05:19 -0400 Subject: [PATCH 4/4] Control max size and count of warning headers Add a static persistent cluster level setting "http.max_warning_header_count" to control the maximum number of warning headers in client HTTP responses. Defaults to unbounded. Add a static persistent cluster level setting "http.max_warning_header_size" to control the maximum total size of warning headers in client HTTP responses. Defaults to unbounded. With every warning header that exceeds these limits, a message will be logged in the main ES log, and any more warning headers for this response will be ignored. --- .../common/util/concurrent/ThreadContext.java | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java index a3946058fe061..901c6425d7131 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java @@ -371,7 +371,6 @@ private static final class ThreadContextStruct { private final Map> responseHeaders; private final boolean isSystemContext; private long warningHeadersSize; //saving current warning headers' size not to recalculate the size with every new warning header - private boolean isWarningLimitReached; private ThreadContextStruct(StreamInput in) throws IOException { final int numRequest = in.readVInt(); Map requestHeaders = numRequest == 0 ? Collections.emptyMap() : new HashMap<>(numRequest); @@ -384,7 +383,6 @@ private ThreadContextStruct(StreamInput in) throws IOException { this.transientHeaders = Collections.emptyMap(); isSystemContext = false; // we never serialize this it's a transient flag this.warningHeadersSize = 0L; - this.isWarningLimitReached = false; } private ThreadContextStruct setSystemContext() { @@ -402,19 +400,17 @@ private ThreadContextStruct(Map requestHeaders, this.transientHeaders = transientHeaders; this.isSystemContext = isSystemContext; this.warningHeadersSize = 0L; - this.isWarningLimitReached = false; } private ThreadContextStruct(Map requestHeaders, Map> responseHeaders, Map transientHeaders, boolean isSystemContext, - long warningHeadersSize, boolean isWarningLimitReached) { + long warningHeadersSize) { this.requestHeaders = requestHeaders; this.responseHeaders = responseHeaders; this.transientHeaders = transientHeaders; this.isSystemContext = isSystemContext; this.warningHeadersSize = warningHeadersSize; - this.isWarningLimitReached = isWarningLimitReached; } /** @@ -473,14 +469,21 @@ private ThreadContextStruct putResponse(final String key, final String value, fi assert value != null; long newWarningHeaderSize = warningHeadersSize; //check if we can add another warning header - if max size within limits - if (key.equals("Warning")) { - if (isWarningLimitReached) return this; // can't add warning headers - limit reached + if (key.equals("Warning") && (maxWarningHeaderSize != -1)) { //if size is NOT unbounded, check its limits + if (warningHeadersSize > maxWarningHeaderSize) { // if max size has already been reached before + final String message = "Dropping a warning header, as their total size reached the maximum allowed of [" + + maxWarningHeaderSize + "] bytes set in [" + + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey() + "]!"; + ESLoggerFactory.getLogger(ThreadContext.class).warn(message); + return this; + } newWarningHeaderSize += "Warning".getBytes(StandardCharsets.UTF_8).length + value.getBytes(StandardCharsets.UTF_8).length; - //if size is NOT unbounded AND limit is exceeded - if ((maxWarningHeaderSize != -1) && (newWarningHeaderSize > maxWarningHeaderSize)) { - logWarningsLimitReached(); - return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, - isSystemContext, newWarningHeaderSize, true); + if (newWarningHeaderSize > maxWarningHeaderSize) { + final String message = "Dropping a warning header, as their total size reached the maximum allowed of [" + + maxWarningHeaderSize + "] bytes set in [" + + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey() + "]!"; + ESLoggerFactory.getLogger(ThreadContext.class).warn(message); + return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, isSystemContext, newWarningHeaderSize); } } @@ -503,21 +506,13 @@ private ThreadContextStruct putResponse(final String key, final String value, fi if ((key.equals("Warning")) && (maxWarningHeaderCount != -1)) { //if count is NOT unbounded, check its limits final int warningHeaderCount = newResponseHeaders.containsKey("Warning") ? newResponseHeaders.get("Warning").size() : 0; if (warningHeaderCount > maxWarningHeaderCount) { - logWarningsLimitReached(); - return new ThreadContextStruct(requestHeaders, responseHeaders, transientHeaders, - isSystemContext, newWarningHeaderSize, true); + final String message = "Dropping a warning header, as their total count reached the maximum allowed of [" + + maxWarningHeaderCount + "] set in [" + HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey() + "]!"; + ESLoggerFactory.getLogger(ThreadContext.class).warn(message); + return this; } } - return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, - isSystemContext, newWarningHeaderSize, isWarningLimitReached); - } - - - private void logWarningsLimitReached() { - final String message = "There were more warnings, but they were dropped as [" + - HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_COUNT.getKey() + "] or [" + - HttpTransportSettings.SETTING_HTTP_MAX_WARNING_HEADER_SIZE.getKey() + "] were reached!"; - ESLoggerFactory.getLogger(ThreadContext.class).warn(message); + return new ThreadContextStruct(requestHeaders, newResponseHeaders, transientHeaders, isSystemContext, newWarningHeaderSize); }