Skip to content

Commit 10c7702

Browse files
github-actions[bot]github-actions[bot]
authored
Merge pull request #160 from elastic/master
🤖 ESQL: Merge upstream
2 parents cab1e64 + c619a20 commit 10c7702

File tree

98 files changed

+1096
-626
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

98 files changed

+1096
-626
lines changed

‎build-tools-internal/src/main/resources/changelog-schema.json

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@
5656
"Infra/Settings",
5757
"Infra/Transport API",
5858
"Ingest",
59+
"Ingest Node",
5960
"Java High Level REST Client",
6061
"Java Low Level REST Client",
6162
"License",

‎docs/changelog/87999.yaml

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 87999
2+
summary: Fix unique realm name check to cover default realms
3+
area: Authentication
4+
type: bug
5+
issues: []

‎docs/changelog/88005.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 88005
2+
summary: "Execute `_refresh` separately from DBQ, with system permissions"
3+
area: Transform
4+
type: bug
5+
issues:
6+
- 88001

‎modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -678,7 +678,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
678678
}
679679
return new NumberFieldMapper.NumericSyntheticFieldLoader(name(), simpleName()) {
680680
@Override
681-
protected void loadNextValue(XContentBuilder b, long value) throws IOException {
681+
protected void writeValue(XContentBuilder b, long value) throws IOException {
682682
b.value(decodeForSyntheticSource(value, scalingFactor));
683683
}
684684
};

‎modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -361,12 +361,12 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
361361
private final Double nullValue = usually() ? null : round(randomValue());
362362

363363
@Override
364-
public SyntheticSourceExample example() {
364+
public SyntheticSourceExample example(int maxValues) {
365365
if (randomBoolean()) {
366366
Tuple<Double, Double> v = generateValue();
367367
return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
368368
}
369-
List<Tuple<Double, Double>> values = randomList(1, 5, this::generateValue);
369+
List<Tuple<Double, Double>> values = randomList(1, maxValues, this::generateValue);
370370
List<Double> in = values.stream().map(Tuple::v1).toList();
371371
List<Double> outList = values.stream().map(Tuple::v2).sorted().toList();
372372
Object out = outList.size() == 1 ? outList.get(0) : outList;

‎modules/reindex/src/main/java/org/elasticsearch/reindex/AbstractBaseReindexRestHandler.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.action.ActionRequestValidationException;
1212
import org.elasticsearch.action.ActionType;
1313
import org.elasticsearch.action.support.ActiveShardCount;
14+
import org.elasticsearch.action.support.ListenableActionFuture;
1415
import org.elasticsearch.client.internal.node.NodeClient;
1516
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
1617
import org.elasticsearch.index.reindex.AbstractBulkByScrollRequest;
@@ -63,7 +64,10 @@ protected RestChannelConsumer doPrepareRequest(RestRequest request, NodeClient c
6364
if (validationException != null) {
6465
throw validationException;
6566
}
66-
return sendTask(client.getLocalNodeId(), client.executeLocally(action, internal, LoggingTaskListener.instance()));
67+
final var responseFuture = new ListenableActionFuture<BulkByScrollResponse>();
68+
final var task = client.executeLocally(action, internal, responseFuture);
69+
responseFuture.addListener(new LoggingTaskListener<>(task));
70+
return sendTask(client.getLocalNodeId(), task);
6771
}
6872

6973
/**

‎server/src/main/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesAction.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
import org.elasticsearch.cluster.metadata.IndexMetadata;
2020
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2121
import org.elasticsearch.cluster.service.ClusterService;
22-
import org.elasticsearch.common.collect.ImmutableOpenMap;
2322
import org.elasticsearch.common.inject.Inject;
2423
import org.elasticsearch.common.logging.DeprecationCategory;
2524
import org.elasticsearch.common.logging.DeprecationLogger;
@@ -95,7 +94,7 @@ protected void masterOperation(Task task, GetAliasesRequest request, ClusterStat
9594
/**
9695
* Fills alias result with empty entries for requested indices when no specific aliases were requested.
9796
*/
98-
static ImmutableOpenMap<String, List<AliasMetadata>> postProcess(
97+
static Map<String, List<AliasMetadata>> postProcess(
9998
GetAliasesRequest request,
10099
String[] concreteIndices,
101100
Map<String, List<AliasMetadata>> aliases,
@@ -105,7 +104,7 @@ static ImmutableOpenMap<String, List<AliasMetadata>> postProcess(
105104
SystemIndices systemIndices
106105
) {
107106
boolean noAliasesSpecified = request.getOriginalAliases() == null || request.getOriginalAliases().length == 0;
108-
ImmutableOpenMap.Builder<String, List<AliasMetadata>> mapBuilder = ImmutableOpenMap.builder(aliases);
107+
Map<String, List<AliasMetadata>> mapBuilder = new HashMap<>(aliases);
109108
for (String index : concreteIndices) {
110109
IndexAbstraction ia = state.metadata().getIndicesLookup().get(index);
111110
assert ia.getType() == IndexAbstraction.Type.CONCRETE_INDEX;
@@ -121,7 +120,7 @@ static ImmutableOpenMap<String, List<AliasMetadata>> postProcess(
121120
assert previous == null;
122121
}
123122
}
124-
final ImmutableOpenMap<String, List<AliasMetadata>> finalResponse = mapBuilder.build();
123+
final Map<String, List<AliasMetadata>> finalResponse = Collections.unmodifiableMap(mapBuilder);
125124
if (systemIndexAccessLevel != SystemIndexAccessLevel.ALL) {
126125
checkSystemIndexAccess(request, systemIndices, state, finalResponse, systemIndexAccessLevel, threadContext);
127126
}
@@ -155,7 +154,7 @@ private static void checkSystemIndexAccess(
155154
GetAliasesRequest request,
156155
SystemIndices systemIndices,
157156
ClusterState state,
158-
ImmutableOpenMap<String, List<AliasMetadata>> aliasesMap,
157+
Map<String, List<AliasMetadata>> aliasesMap,
159158
SystemIndexAccessLevel systemIndexAccessLevel,
160159
ThreadContext threadContext
161160
) {

‎server/src/main/java/org/elasticsearch/action/admin/indices/get/TransportGetIndexAction.java

+15-15
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@
2929
import org.elasticsearch.threadpool.ThreadPool;
3030
import org.elasticsearch.transport.TransportService;
3131

32+
import java.util.Collections;
33+
import java.util.HashMap;
3234
import java.util.List;
3335
import java.util.Map;
3436
import java.util.stream.Collectors;
@@ -78,17 +80,15 @@ protected void doMasterOperation(
7880
) {
7981
Map<String, MappingMetadata> mappingsResult = ImmutableOpenMap.of();
8082
Map<String, List<AliasMetadata>> aliasesResult = Map.of();
81-
ImmutableOpenMap<String, Settings> settings = ImmutableOpenMap.of();
82-
ImmutableOpenMap<String, Settings> defaultSettings = ImmutableOpenMap.of();
83-
ImmutableOpenMap<String, String> dataStreams = ImmutableOpenMap.<String, String>builder()
84-
.putAllFromMap(
85-
state.metadata()
86-
.findDataStreams(concreteIndices)
87-
.entrySet()
88-
.stream()
89-
.collect(Collectors.toMap(Map.Entry::getKey, v -> v.getValue().getName()))
90-
)
91-
.build();
83+
Map<String, Settings> settings = Map.of();
84+
Map<String, Settings> defaultSettings = Map.of();
85+
Map<String, String> dataStreams = Map.copyOf(
86+
state.metadata()
87+
.findDataStreams(concreteIndices)
88+
.entrySet()
89+
.stream()
90+
.collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, v -> v.getValue().getName()))
91+
);
9292
Feature[] features = request.features();
9393
boolean doneAliases = false;
9494
boolean doneMappings = false;
@@ -111,8 +111,8 @@ protected void doMasterOperation(
111111
break;
112112
case SETTINGS:
113113
if (doneSettings == false) {
114-
ImmutableOpenMap.Builder<String, Settings> settingsMapBuilder = ImmutableOpenMap.builder();
115-
ImmutableOpenMap.Builder<String, Settings> defaultSettingsMapBuilder = ImmutableOpenMap.builder();
114+
Map<String, Settings> settingsMapBuilder = new HashMap<>();
115+
Map<String, Settings> defaultSettingsMapBuilder = new HashMap<>();
116116
for (String index : concreteIndices) {
117117
checkCancellation(task);
118118
Settings indexSettings = state.metadata().index(index).getSettings();
@@ -127,8 +127,8 @@ protected void doMasterOperation(
127127
defaultSettingsMapBuilder.put(index, defaultIndexSettings);
128128
}
129129
}
130-
settings = settingsMapBuilder.build();
131-
defaultSettings = defaultSettingsMapBuilder.build();
130+
settings = Collections.unmodifiableMap(settingsMapBuilder);
131+
defaultSettings = Collections.unmodifiableMap(defaultSettingsMapBuilder);
132132
doneSettings = true;
133133
}
134134
break;

‎server/src/main/java/org/elasticsearch/action/admin/indices/settings/get/TransportGetSettingsAction.java

+10-6
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
import org.elasticsearch.cluster.metadata.IndexMetadata;
1818
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
1919
import org.elasticsearch.cluster.service.ClusterService;
20-
import org.elasticsearch.common.collect.ImmutableOpenMap;
2120
import org.elasticsearch.common.inject.Inject;
2221
import org.elasticsearch.common.regex.Regex;
2322
import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -29,6 +28,11 @@
2928
import org.elasticsearch.threadpool.ThreadPool;
3029
import org.elasticsearch.transport.TransportService;
3130

31+
import java.util.HashMap;
32+
import java.util.Map;
33+
34+
import static java.util.Collections.unmodifiableMap;
35+
3236
public class TransportGetSettingsAction extends TransportMasterNodeReadAction<GetSettingsRequest, GetSettingsResponse> {
3337

3438
private final SettingsFilter settingsFilter;
@@ -77,8 +81,8 @@ protected void masterOperation(
7781
ActionListener<GetSettingsResponse> listener
7882
) {
7983
Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
80-
ImmutableOpenMap.Builder<String, Settings> indexToSettingsBuilder = ImmutableOpenMap.builder();
81-
ImmutableOpenMap.Builder<String, Settings> indexToDefaultSettingsBuilder = ImmutableOpenMap.builder();
84+
Map<String, Settings> indexToSettings = new HashMap<>();
85+
Map<String, Settings> indexToDefaultSettings = new HashMap<>();
8286
for (Index concreteIndex : concreteIndices) {
8387
IndexMetadata indexMetadata = state.getMetadata().index(concreteIndex);
8488
if (indexMetadata == null) {
@@ -94,15 +98,15 @@ protected void masterOperation(
9498
indexSettings = indexSettings.filter(k -> Regex.simpleMatch(request.names(), k));
9599
}
96100

97-
indexToSettingsBuilder.put(concreteIndex.getName(), indexSettings);
101+
indexToSettings.put(concreteIndex.getName(), indexSettings);
98102
if (request.includeDefaults()) {
99103
Settings defaultSettings = settingsFilter.filter(indexScopedSettings.diff(indexSettings, Settings.EMPTY));
100104
if (isFilteredRequest(request)) {
101105
defaultSettings = defaultSettings.filter(k -> Regex.simpleMatch(request.names(), k));
102106
}
103-
indexToDefaultSettingsBuilder.put(concreteIndex.getName(), defaultSettings);
107+
indexToDefaultSettings.put(concreteIndex.getName(), defaultSettings);
104108
}
105109
}
106-
listener.onResponse(new GetSettingsResponse(indexToSettingsBuilder.build(), indexToDefaultSettingsBuilder.build()));
110+
listener.onResponse(new GetSettingsResponse(unmodifiableMap(indexToSettings), unmodifiableMap(indexToDefaultSettings)));
107111
}
108112
}

‎server/src/main/java/org/elasticsearch/client/internal/node/NodeClient.java

+6-21
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import org.elasticsearch.common.settings.Settings;
2121
import org.elasticsearch.tasks.Task;
2222
import org.elasticsearch.tasks.TaskCancelledException;
23-
import org.elasticsearch.tasks.TaskListener;
2423
import org.elasticsearch.tasks.TaskManager;
2524
import org.elasticsearch.threadpool.ThreadPool;
2625
import org.elasticsearch.transport.RemoteClusterService;
@@ -41,7 +40,7 @@ public class NodeClient extends AbstractClient {
4140

4241
/**
4342
* The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
44-
* {@link #executeLocally(ActionType, ActionRequest, TaskListener)}.
43+
* {@link #executeLocally(ActionType, ActionRequest, ActionListener)}.
4544
*/
4645
private Supplier<String> localNodeId;
4746
private Transport.Connection localConnection;
@@ -115,27 +114,13 @@ public <Request extends ActionRequest, Response extends ActionResponse> Task exe
115114
transportAction(action),
116115
request,
117116
localConnection,
118-
new ActionResponseTaskListener<>(listener)
117+
new SafelyWrappedActionListener<>(listener)
119118
);
120119
}
121120

122-
/**
123-
* Execute an {@link ActionType} locally, returning that {@link Task} used to track it, and linking an {@link TaskListener}.
124-
* Prefer this method if you need access to the task when listening for the response.
125-
*
126-
* @throws TaskCancelledException if the request's parent task has been cancelled already
127-
*/
128-
public <Request extends ActionRequest, Response extends ActionResponse> Task executeLocally(
129-
ActionType<Response> action,
130-
Request request,
131-
TaskListener<Response> listener
132-
) {
133-
return taskManager.registerAndExecute("transport", transportAction(action), request, localConnection, listener);
134-
}
135-
136121
/**
137122
* The id of the local {@link DiscoveryNode}. Useful for generating task ids from tasks returned by
138-
* {@link #executeLocally(ActionType, ActionRequest, TaskListener)}.
123+
* {@link #executeLocally(ActionType, ActionRequest, ActionListener)}.
139124
*/
140125
public String getLocalNodeId() {
141126
return localNodeId.get();
@@ -167,10 +152,10 @@ public NamedWriteableRegistry getNamedWriteableRegistry() {
167152
return namedWriteableRegistry;
168153
}
169154

170-
private record ActionResponseTaskListener<Response> (ActionListener<Response> listener) implements TaskListener<Response> {
155+
private record SafelyWrappedActionListener<Response> (ActionListener<Response> listener) implements ActionListener<Response> {
171156

172157
@Override
173-
public void onResponse(Task task, Response response) {
158+
public void onResponse(Response response) {
174159
try {
175160
listener.onResponse(response);
176161
} catch (Exception e) {
@@ -180,7 +165,7 @@ public void onResponse(Task task, Response response) {
180165
}
181166

182167
@Override
183-
public void onFailure(Task task, Exception e) {
168+
public void onFailure(Exception e) {
184169
try {
185170
listener.onFailure(e);
186171
} catch (Exception ex) {

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/FilterAllocationDecider.java

+4-6
Original file line numberDiff line numberDiff line change
@@ -90,15 +90,13 @@ public FilterAllocationDecider(Settings settings, ClusterSettings clusterSetting
9090

9191
@Override
9292
public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
93-
if (shardRouting.unassigned()) {
93+
if (shardRouting.unassigned() && shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
9494
// only for unassigned - we filter allocation right after the index creation (for shard shrinking) to ensure
9595
// that once it has been allocated post API the replicas can be allocated elsewhere without user interaction
9696
// this is a setting that can only be set within the system!
97-
IndexMetadata indexMd = allocation.metadata().getIndexSafe(shardRouting.index());
98-
DiscoveryNodeFilters initialRecoveryFilters = DiscoveryNodeFilters.trimTier(indexMd.getInitialRecoveryFilters());
99-
if (initialRecoveryFilters != null
100-
&& shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS
101-
&& initialRecoveryFilters.match(node.node()) == false) {
97+
IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
98+
DiscoveryNodeFilters initialRecoveryFilters = DiscoveryNodeFilters.trimTier(indexMetadata.getInitialRecoveryFilters());
99+
if (initialRecoveryFilters != null && initialRecoveryFilters.match(node.node()) == false) {
102100
String explanation =
103101
"initial allocation of the shrunken index is only allowed on nodes [%s] that hold a copy of every shard in the index";
104102
return allocation.decision(Decision.NO, NAME, explanation, initialRecoveryFilters);

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/ResizeAllocationDecider.java

+4-6
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@
1212
import org.elasticsearch.cluster.routing.RecoverySource;
1313
import org.elasticsearch.cluster.routing.RoutingNode;
1414
import org.elasticsearch.cluster.routing.ShardRouting;
15-
import org.elasticsearch.cluster.routing.UnassignedInfo;
1615
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
1716
import org.elasticsearch.index.Index;
1817
import org.elasticsearch.index.shard.ShardId;
@@ -31,16 +30,15 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingAllocation allocat
3130

3231
@Override
3332
public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
34-
final UnassignedInfo unassignedInfo = shardRouting.unassignedInfo();
35-
if (unassignedInfo != null && shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
33+
if (shardRouting.unassignedInfo() != null && shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS) {
3634
// we only make decisions here if we have an unassigned info and we have to recover from another index ie. split / shrink
3735
final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
38-
Index resizeSourceIndex = indexMetadata.getResizeSourceIndex();
36+
final Index resizeSourceIndex = indexMetadata.getResizeSourceIndex();
3937
assert resizeSourceIndex != null;
40-
if (allocation.metadata().index(resizeSourceIndex) == null) {
38+
final IndexMetadata sourceIndexMetadata = allocation.metadata().index(resizeSourceIndex);
39+
if (sourceIndexMetadata == null) {
4140
return allocation.decision(Decision.NO, NAME, "resize source index [%s] doesn't exists", resizeSourceIndex.toString());
4241
}
43-
IndexMetadata sourceIndexMetadata = allocation.metadata().getIndexSafe(resizeSourceIndex);
4442
if (indexMetadata.getNumberOfShards() < sourceIndexMetadata.getNumberOfShards()) {
4543
// this only handles splits and clone so far.
4644
return Decision.ALWAYS;

‎server/src/main/java/org/elasticsearch/common/logging/LogConfigurator.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ public static void configure(final Environment environment, boolean useConsole)
124124
configure(environment.settings(), environment.configFile(), environment.logsFile(), useConsole);
125125
}
126126

127-
private static void configureESLogging() {
127+
public static void configureESLogging() {
128128
LoggerFactory.setInstance(new LoggerFactoryImpl());
129129
}
130130

‎server/src/main/java/org/elasticsearch/index/get/ShardGetService.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ private GetResult innerGetFetch(
257257
SourceLoader loader = forceSyntheticSource
258258
? new SourceLoader.Synthetic(mappingLookup.getMapping())
259259
: mappingLookup.newSourceLoader();
260-
source = loader.leaf(docIdAndVersion.reader).source(fieldVisitor, docIdAndVersion.docId);
260+
source = loader.leaf(docIdAndVersion.reader, new int[] { docIdAndVersion.docId }).source(fieldVisitor, docIdAndVersion.docId);
261261

262262
// put stored fields into result objects
263263
if (fieldVisitor.fields().isEmpty() == false) {

‎server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -439,7 +439,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
439439
}
440440
return new NumberFieldMapper.NumericSyntheticFieldLoader(name(), simpleName()) {
441441
@Override
442-
protected void loadNextValue(XContentBuilder b, long value) throws IOException {
442+
protected void writeValue(XContentBuilder b, long value) throws IOException {
443443
b.value(value == 1);
444444
}
445445
};

‎server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -917,7 +917,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
917917
}
918918
return new NumberFieldMapper.NumericSyntheticFieldLoader(name(), simpleName()) {
919919
@Override
920-
protected void loadNextValue(XContentBuilder b, long value) throws IOException {
920+
protected void writeValue(XContentBuilder b, long value) throws IOException {
921921
b.value(fieldType().format(value, fieldType().dateTimeFormatter()));
922922
}
923923
};

0 commit comments

Comments
 (0)