Skip to content

Commit 7d2e7f4

Browse files
jasontedorjasontedor
committed
Merge branch 'master' into check-for-close
* master: Fix handling of paths containing parentheses Allow only a fixed-size receive predictor (elastic#26165) Add Homebrew instructions to getting started ingest: Fix bug that prevent date_index_name processor from accepting timestamps specified as a json number Scripting: Fix expressions to temporarily support filter scripts (elastic#26824) Docs: Add note to contributing docs warning against tool based refactoring (elastic#26936) Fix thread context handling of headers overriding (elastic#26068) SearchWhileCreatingIndexIT: remove usage of _only_nodes
2 parents 36c2753 + c03f0c8 commit 7d2e7f4

File tree

12 files changed

+91
-54
lines changed

12 files changed

+91
-54
lines changed

‎CONTRIBUTING.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,11 @@ If you have a bugfix or new feature that you would like to contribute to Elastic
3838

3939
We enjoy working with contributors to get their code accepted. There are many approaches to fixing a problem and it is important to find the best approach before writing too much code.
4040

41+
Note that it is unlikely the project will merge refactors for the sake of refactoring. These
42+
types of pull requests have a high cost to maintainers in reviewing and testing with little
43+
to no tangible benefit. This especially includes changes generated by tools. For example,
44+
converting all generic interface instances to use the diamond operator.
45+
4146
The process for contributing to any of the [Elastic repositories](https://github.com/elastic/) is similar. Details for individual projects can be found below.
4247

4348
### Fork and clone the repository

‎core/src/main/java/org/elasticsearch/common/util/concurrent/ThreadContext.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -407,11 +407,10 @@ private ThreadContextStruct putHeaders(Map<String, String> headers) {
407407
if (headers.isEmpty()) {
408408
return this;
409409
} else {
410-
final Map<String, String> newHeaders = new HashMap<>();
410+
final Map<String, String> newHeaders = new HashMap<>(this.requestHeaders);
411411
for (Map.Entry<String, String> entry : headers.entrySet()) {
412412
putSingleHeader(entry.getKey(), entry.getValue(), newHeaders);
413413
}
414-
newHeaders.putAll(this.requestHeaders);
415414
return new ThreadContextStruct(newHeaders, responseHeaders, transientHeaders, isSystemContext);
416415
}
417416
}

‎core/src/test/java/org/elasticsearch/common/util/concurrent/ThreadContextTests.java

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import java.util.List;
3030
import java.util.Map;
3131
import java.util.function.Supplier;
32-
3332
import static org.hamcrest.Matchers.equalTo;
3433
import static org.hamcrest.Matchers.hasItem;
3534
import static org.hamcrest.Matchers.hasSize;
@@ -215,8 +214,8 @@ public void testResponseHeaders() {
215214
public void testCopyHeaders() {
216215
Settings build = Settings.builder().put("request.headers.default", "1").build();
217216
ThreadContext threadContext = new ThreadContext(build);
218-
threadContext.copyHeaders(Collections.<String,String>emptyMap().entrySet());
219-
threadContext.copyHeaders(Collections.<String,String>singletonMap("foo", "bar").entrySet());
217+
threadContext.copyHeaders(Collections.<String, String>emptyMap().entrySet());
218+
threadContext.copyHeaders(Collections.<String, String>singletonMap("foo", "bar").entrySet());
220219
assertEquals("bar", threadContext.getHeader("foo"));
221220
}
222221

@@ -443,7 +442,7 @@ public void onAfter() {
443442
assertEquals("bar", threadContext.getHeader("foo"));
444443
assertEquals("bar_transient", threadContext.getTransient("foo"));
445444
assertNotNull(threadContext.getTransient("failure"));
446-
assertEquals("exception from doRun", ((RuntimeException)threadContext.getTransient("failure")).getMessage());
445+
assertEquals("exception from doRun", ((RuntimeException) threadContext.getTransient("failure")).getMessage());
447446
assertFalse(threadContext.isDefaultContext());
448447
threadContext.putTransient("after", "after");
449448
}
@@ -604,7 +603,7 @@ protected void doRun() throws Exception {
604603
public void testMarkAsSystemContext() throws IOException {
605604
try (ThreadContext threadContext = new ThreadContext(Settings.EMPTY)) {
606605
assertFalse(threadContext.isSystemContext());
607-
try(ThreadContext.StoredContext context = threadContext.stashContext()){
606+
try (ThreadContext.StoredContext context = threadContext.stashContext()) {
608607
assertFalse(threadContext.isSystemContext());
609608
threadContext.markAsSystemContext();
610609
assertTrue(threadContext.isSystemContext());
@@ -613,6 +612,17 @@ public void testMarkAsSystemContext() throws IOException {
613612
}
614613
}
615614

615+
public void testPutHeaders() {
616+
Settings build = Settings.builder().put("request.headers.default", "1").build();
617+
ThreadContext threadContext = new ThreadContext(build);
618+
threadContext.putHeader(Collections.<String, String>emptyMap());
619+
threadContext.putHeader(Collections.<String, String>singletonMap("foo", "bar"));
620+
assertEquals("bar", threadContext.getHeader("foo"));
621+
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () ->
622+
threadContext.putHeader(Collections.<String, String>singletonMap("foo", "boom")));
623+
assertEquals("value for key [foo] already present", e.getMessage());
624+
}
625+
616626
/**
617627
* Sometimes wraps a Runnable in an AbstractRunnable.
618628
*/

‎core/src/test/java/org/elasticsearch/search/basic/SearchWhileCreatingIndexIT.java

Lines changed: 5 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.elasticsearch.action.search.SearchResponse;
2424
import org.elasticsearch.client.Client;
2525
import org.elasticsearch.cluster.health.ClusterHealthStatus;
26-
import org.elasticsearch.cluster.routing.IndexShardRoutingTable;
2726
import org.elasticsearch.index.query.QueryBuilders;
2827
import org.elasticsearch.test.ESIntegTestCase;
2928
import org.elasticsearch.test.junit.annotations.TestLogging;
@@ -74,21 +73,13 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas)
7473

7574
logger.info("using preference {}", preference);
7675
// we want to make sure that while recovery happens, and a replica gets recovered, its properly refreshed
77-
ClusterHealthStatus status = client().admin().cluster().prepareHealth("test").get().getStatus();
78-
76+
ClusterHealthStatus status = client().admin().cluster().prepareHealth("test").get().getStatus();;
7977
while (status != ClusterHealthStatus.GREEN) {
80-
// first, verify that search on the primary search works
81-
for (IndexShardRoutingTable shardRoutingTable : clusterService().state().routingTable().index("test")) {
82-
String primaryNode = shardRoutingTable.primaryShard().currentNodeId();
83-
SearchResponse searchResponse = client().prepareSearch("test")
84-
.setPreference("_only_nodes:" + primaryNode)
85-
.setQuery(QueryBuilders.termQuery("field", "test"))
86-
.execute().actionGet();
87-
assertHitCount(searchResponse, 1);
88-
break;
89-
}
78+
// first, verify that search normal search works
79+
SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
80+
assertHitCount(searchResponse, 1);
9081
Client client = client();
91-
SearchResponse searchResponse = client.prepareSearch("test").setPreference(preference + Integer.toString(counter++)).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
82+
searchResponse = client.prepareSearch("test").setPreference(preference + Integer.toString(counter++)).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
9283
if (searchResponse.getHits().getTotalHits() != 1) {
9384
refresh();
9485
SearchResponse searchResponseAfterRefresh = client.prepareSearch("test").setPreference(preference).setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
@@ -102,13 +93,6 @@ private void searchWhileCreatingIndex(boolean createIndex, int numberOfReplicas)
10293
status = client().admin().cluster().prepareHealth("test").get().getStatus();
10394
internalCluster().ensureAtLeastNumDataNodes(numberOfReplicas + 1);
10495
}
105-
106-
for (String node : internalCluster().nodesInclude("test")) {
107-
SearchResponse searchResponse = client().prepareSearch("test")
108-
.setPreference("_prefer_nodes:" + node)
109-
.setQuery(QueryBuilders.termQuery("field", "test")).execute().actionGet();
110-
assertHitCount(searchResponse, 1);
111-
}
11296
cluster().wipeIndices("test");
11397
}
11498
}

‎distribution/src/main/resources/bin/elasticsearch-service.bat

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -163,15 +163,15 @@ for %%a in ("%ES_JAVA_OPTS:;=","%") do (
163163
@endlocal & set JVM_MS=%JVM_MS% & set JVM_MX=%JVM_MX% & set JVM_SS=%JVM_SS%
164164

165165
if "%JVM_MS%" == "" (
166-
echo minimum heap size not set; configure using -Xms via %ES_JVM_OPTIONS% or ES_JAVA_OPTS
166+
echo minimum heap size not set; configure using -Xms via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
167167
goto:eof
168168
)
169169
if "%JVM_MX%" == "" (
170-
echo maximum heap size not set; configure using -Xmx via %ES_JVM_OPTIONS% or ES_JAVA_OPTS
170+
echo maximum heap size not set; configure using -Xmx via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
171171
goto:eof
172172
)
173173
if "%JVM_SS%" == "" (
174-
echo thread stack size not set; configure using -Xss via %ES_JVM_OPTIONS% or ES_JAVA_OPTS
174+
echo thread stack size not set; configure using -Xss via "%ES_JVM_OPTIONS%" or ES_JAVA_OPTS
175175
goto:eof
176176
)
177177

‎docs/reference/getting-started.asciidoc

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,16 @@ And now we are ready to start our node and single cluster:
148148
./elasticsearch
149149
--------------------------------------------------
150150

151+
[float]
152+
=== Instalation with Homebrew
153+
154+
On macOS, Elasticsearch can also be installed via https://brew.sh[Homebrew]:
155+
156+
["source","sh"]
157+
--------------------------------------------------
158+
brew install elasticsearch
159+
--------------------------------------------------
160+
151161
[float]
152162
=== Installation example with MSI Windows Installer
153163

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.util.List;
2626
import java.util.Locale;
2727
import java.util.Map;
28+
import java.util.Objects;
2829
import java.util.function.Function;
2930

3031
import org.elasticsearch.ExceptionsHelper;
@@ -61,7 +62,8 @@ public final class DateIndexNameProcessor extends AbstractProcessor {
6162

6263
@Override
6364
public void execute(IngestDocument ingestDocument) throws Exception {
64-
String date = ingestDocument.getFieldValue(field, String.class);
65+
// Date can be specified as a string or long:
66+
String date = Objects.toString(ingestDocument.getFieldValue(field, Object.class));
6567

6668
DateTime dateTime = null;
6769
Exception lastException = null;

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,11 @@ public void testUnixMs()throws Exception {
6262
Collections.singletonMap("_field", "1000500"));
6363
dateProcessor.execute(document);
6464
assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
65+
66+
document = new IngestDocument("_index", "_type", "_id", null, null,
67+
Collections.singletonMap("_field", 1000500L));
68+
dateProcessor.execute(document);
69+
assertThat(document.getSourceAndMetadata().get("_index"), equalTo("<events-{19700101||/m{yyyyMMdd|UTC}}>"));
6570
}
6671

6772
public void testUnix()throws Exception {

‎modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.elasticsearch.index.mapper.MapperService;
3939
import org.elasticsearch.script.ClassPermission;
4040
import org.elasticsearch.script.ExecutableScript;
41+
import org.elasticsearch.script.FilterScript;
4142
import org.elasticsearch.script.ScriptContext;
4243
import org.elasticsearch.script.ScriptEngine;
4344
import org.elasticsearch.script.ScriptException;
@@ -107,6 +108,9 @@ protected Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundE
107108
} else if (context.instanceClazz.equals(ExecutableScript.class)) {
108109
ExecutableScript.Factory factory = (p) -> new ExpressionExecutableScript(expr, p);
109110
return context.factoryClazz.cast(factory);
111+
} else if (context.instanceClazz.equals(FilterScript.class)) {
112+
FilterScript.Factory factory = (p, lookup) -> newFilterScript(expr, lookup, p);
113+
return context.factoryClazz.cast(factory);
110114
}
111115
throw new IllegalArgumentException("expression engine does not know how to handle script context [" + context.name + "]");
112116
}
@@ -236,6 +240,27 @@ private SearchScript.LeafFactory newSearchScript(Expression expr, SearchLookup l
236240
return new ExpressionSearchScript(expr, bindings, specialValue, needsScores);
237241
}
238242

243+
/**
244+
* This is a hack for filter scripts, which must return booleans instead of doubles as expression do.
245+
* See https://github.com/elastic/elasticsearch/issues/26429.
246+
*/
247+
private FilterScript.LeafFactory newFilterScript(Expression expr, SearchLookup lookup, @Nullable Map<String, Object> vars) {
248+
SearchScript.LeafFactory searchLeafFactory = newSearchScript(expr, lookup, vars);
249+
return ctx -> {
250+
SearchScript script = searchLeafFactory.newInstance(ctx);
251+
return new FilterScript(vars, lookup, ctx) {
252+
@Override
253+
public boolean execute() {
254+
return script.runAsDouble() != 0.0;
255+
}
256+
@Override
257+
public void setDocument(int docid) {
258+
script.setDocument(docid);
259+
}
260+
};
261+
};
262+
}
263+
239264
/**
240265
* converts a ParseException at compile-time or link-time to a ScriptException
241266
*/

‎modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -700,4 +700,19 @@ public void testBoolean() throws Exception {
700700
assertEquals(2.0D, rsp.getHits().getAt(1).field("foo").getValue(), 1.0D);
701701
assertEquals(2.0D, rsp.getHits().getAt(2).field("foo").getValue(), 1.0D);
702702
}
703+
704+
public void testFilterScript() throws Exception {
705+
createIndex("test");
706+
ensureGreen("test");
707+
indexRandom(true,
708+
client().prepareIndex("test", "doc", "1").setSource("foo", 1.0),
709+
client().prepareIndex("test", "doc", "2").setSource("foo", 0.0));
710+
SearchRequestBuilder builder = buildRequest("doc['foo'].value");
711+
Script script = new Script(ScriptType.INLINE, "expression", "doc['foo'].value", Collections.emptyMap());
712+
builder.setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.scriptQuery(script)));
713+
SearchResponse rsp = builder.get();
714+
assertSearchResponse(rsp);
715+
assertEquals(1, rsp.getHits().getTotalHits());
716+
assertEquals(1.0D, rsp.getHits().getAt(0).field("foo").getValue(), 0.0D);
717+
}
703718
}

‎modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import com.carrotsearch.hppc.IntHashSet;
2323
import com.carrotsearch.hppc.IntSet;
2424
import io.netty.bootstrap.ServerBootstrap;
25-
import io.netty.channel.AdaptiveRecvByteBufAllocator;
2625
import io.netty.channel.Channel;
2726
import io.netty.channel.ChannelFuture;
2827
import io.netty.channel.ChannelHandler;
@@ -88,7 +87,6 @@
8887
import java.util.regex.Pattern;
8988

9089
import static org.elasticsearch.common.settings.Setting.boolSetting;
91-
import static org.elasticsearch.common.settings.Setting.byteSizeSetting;
9290
import static org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory;
9391
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_CREDENTIALS;
9492
import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_HEADERS;
@@ -132,10 +130,6 @@ public class Netty4HttpServerTransport extends AbstractLifecycleComponent implem
132130

133131
public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE =
134132
Setting.byteSizeSetting("http.netty.receive_predictor_size", new ByteSizeValue(64, ByteSizeUnit.KB), Property.NodeScope);
135-
public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN =
136-
byteSizeSetting("http.netty.receive_predictor_min", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
137-
public static final Setting<ByteSizeValue> SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX =
138-
byteSizeSetting("http.netty.receive_predictor_max", SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE, Property.NodeScope);
139133

140134

141135
protected final NetworkService networkService;
@@ -227,17 +221,8 @@ public Netty4HttpServerTransport(Settings settings, NetworkService networkServic
227221
this.tcpReceiveBufferSize = SETTING_HTTP_TCP_RECEIVE_BUFFER_SIZE.get(settings);
228222
this.detailedErrorsEnabled = SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings);
229223

230-
// See AdaptiveReceiveBufferSizePredictor#DEFAULT_XXX for default values in netty..., we can use higher ones for us, even fixed one
231-
ByteSizeValue receivePredictorMin = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN.get(settings);
232-
ByteSizeValue receivePredictorMax = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX.get(settings);
233-
if (receivePredictorMax.getBytes() == receivePredictorMin.getBytes()) {
234-
recvByteBufAllocator = new FixedRecvByteBufAllocator(Math.toIntExact(receivePredictorMax.getBytes()));
235-
} else {
236-
recvByteBufAllocator = new AdaptiveRecvByteBufAllocator(
237-
Math.toIntExact(receivePredictorMin.getBytes()),
238-
Math.toIntExact(receivePredictorMin.getBytes()),
239-
Math.toIntExact(receivePredictorMax.getBytes()));
240-
}
224+
ByteSizeValue receivePredictor = SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE.get(settings);
225+
recvByteBufAllocator = new FixedRecvByteBufAllocator(receivePredictor.bytesAsInt());
241226

242227
this.compression = SETTING_HTTP_COMPRESSION.get(settings);
243228
this.compressionLevel = SETTING_HTTP_COMPRESSION_LEVEL.get(settings);
@@ -253,9 +238,8 @@ public Netty4HttpServerTransport(Settings settings, NetworkService networkServic
253238
this.maxContentLength = maxContentLength;
254239

255240
logger.debug("using max_chunk_size[{}], max_header_size[{}], max_initial_line_length[{}], max_content_length[{}], " +
256-
"receive_predictor[{}->{}], pipelining[{}], pipelining_max_events[{}]",
257-
maxChunkSize, maxHeaderSize, maxInitialLineLength, this.maxContentLength,
258-
receivePredictorMin, receivePredictorMax, pipelining, pipeliningMaxEvents);
241+
"receive_predictor[{}], pipelining[{}], pipelining_max_events[{}]",
242+
maxChunkSize, maxHeaderSize, maxInitialLineLength, this.maxContentLength, receivePredictor, pipelining, pipeliningMaxEvents);
259243
}
260244

261245
public Settings settings() {

‎modules/transport-netty4/src/main/java/org/elasticsearch/transport/Netty4Plugin.java

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ public List<Setting<?>> getSettings() {
5656
Netty4HttpServerTransport.SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS,
5757
Netty4HttpServerTransport.SETTING_HTTP_WORKER_COUNT,
5858
Netty4HttpServerTransport.SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_SIZE,
59-
Netty4HttpServerTransport.SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MIN,
60-
Netty4HttpServerTransport.SETTING_HTTP_NETTY_RECEIVE_PREDICTOR_MAX,
6159
Netty4Transport.WORKER_COUNT,
6260
Netty4Transport.NETTY_RECEIVE_PREDICTOR_SIZE,
6361
Netty4Transport.NETTY_RECEIVE_PREDICTOR_MIN,

0 commit comments

Comments
 (0)