Skip to content

Commit 6d2cab9

Browse files
nik9000nik9000
authored
Stop runtime script from emitting too many values (#61938) (#62186)
This prevent `keyword` valued runtime scripts from emitting too many values or values that take up too much space. Without this you can put allocate a ton of memory with the script by sticking it into a tight loop. Painless has some protections against this but: 1. I don't want to rely on them out of sheer paranoia 2. They don't really kick in when the script uses callbacks like we do anyway. Relates to #59332
1 parent 1eb4595 commit 6d2cab9

27 files changed

+396
-47
lines changed

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ public abstract class AbstractLongScriptFieldScript extends AbstractScriptFieldS
1919
private long[] values = new long[1];
2020
private int count;
2121

22-
public AbstractLongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
23-
super(params, searchLookup, ctx);
22+
public AbstractLongScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
23+
super(fieldName, params, searchLookup, ctx);
2424
}
2525

2626
/**
@@ -50,6 +50,7 @@ public final int count() {
5050
}
5151

5252
protected final void emitValue(long v) {
53+
checkMaxSize(count);
5354
if (values.length < count + 1) {
5455
values = ArrayUtil.grow(values, count + 1);
5556
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java

+27-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.search.lookup.SourceLookup;
1818

1919
import java.util.HashMap;
20+
import java.util.Locale;
2021
import java.util.Map;
2122
import java.util.function.Function;
2223

@@ -27,6 +28,11 @@
2728
* {@link AggregationScript} but hopefully with less historical baggage.
2829
*/
2930
public abstract class AbstractScriptFieldScript {
31+
/**
32+
* The maximum number of values a script should be allowed to emit.
33+
*/
34+
static final int MAX_VALUES = 100;
35+
3036
public static <F> ScriptContext<F> newContext(String name, Class<F> factoryClass) {
3137
return new ScriptContext<F>(
3238
name + "_script_field",
@@ -54,10 +60,12 @@ public static <F> ScriptContext<F> newContext(String name, Class<F> factoryClass
5460
value -> ((SourceLookup) value).loadSourceIfNeeded()
5561
);
5662

63+
protected final String fieldName;
5764
private final Map<String, Object> params;
5865
private final LeafSearchLookup leafSearchLookup;
5966

60-
public AbstractScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
67+
public AbstractScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
68+
this.fieldName = fieldName;
6169
this.leafSearchLookup = searchLookup.getLeafSearchLookup(ctx);
6270
params = new HashMap<>(params);
6371
params.put("_source", leafSearchLookup.source());
@@ -94,5 +102,23 @@ public final Map<String, ScriptDocValues<?>> getDoc() {
94102
return leafSearchLookup.doc();
95103
}
96104

105+
/**
106+
* Check if the we can add another value to the list of values.
107+
* @param currentSize the current size of the list
108+
*/
109+
protected final void checkMaxSize(int currentSize) {
110+
if (currentSize >= MAX_VALUES) {
111+
throw new IllegalArgumentException(
112+
String.format(
113+
Locale.ROOT,
114+
"Runtime field [%s] is emitting [%s] values while the maximum number of values allowed is [%s]",
115+
fieldName,
116+
currentSize + 1,
117+
MAX_VALUES
118+
)
119+
);
120+
}
121+
}
122+
97123
public abstract void execute();
98124
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScript.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ static List<Whitelist> whitelist() {
3131
public static final String[] PARAMETERS = {};
3232

3333
public interface Factory extends ScriptFactory {
34-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
34+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
3535
}
3636

3737
public interface LeafFactory {
@@ -41,8 +41,8 @@ public interface LeafFactory {
4141
private int trues;
4242
private int falses;
4343

44-
public BooleanScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
45-
super(params, searchLookup, ctx);
44+
public BooleanScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
45+
super(fieldName, params, searchLookup, ctx);
4646
}
4747

4848
/**

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java

+9-3
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ static List<Whitelist> whitelist() {
3131
public static final String[] PARAMETERS = {};
3232

3333
public interface Factory extends ScriptFactory {
34-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter);
34+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter);
3535
}
3636

3737
public interface LeafFactory {
@@ -40,8 +40,14 @@ public interface LeafFactory {
4040

4141
private final DateFormatter formatter;
4242

43-
public DateScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter, LeafReaderContext ctx) {
44-
super(params, searchLookup, ctx);
43+
public DateScriptFieldScript(
44+
String fieldName,
45+
Map<String, Object> params,
46+
SearchLookup searchLookup,
47+
DateFormatter formatter,
48+
LeafReaderContext ctx
49+
) {
50+
super(fieldName, params, searchLookup, ctx);
4551
this.formatter = formatter;
4652
}
4753

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ static List<Whitelist> whitelist() {
3131
public static final String[] PARAMETERS = {};
3232

3333
public interface Factory extends ScriptFactory {
34-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
34+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
3535
}
3636

3737
public interface LeafFactory {
@@ -41,8 +41,8 @@ public interface LeafFactory {
4141
private double[] values = new double[1];
4242
private int count;
4343

44-
public DoubleScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
45-
super(params, searchLookup, ctx);
44+
public DoubleScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
45+
super(fieldName, params, searchLookup, ctx);
4646
}
4747

4848
/**
@@ -72,6 +72,7 @@ public final int count() {
7272
}
7373

7474
protected final void emitValue(double v) {
75+
checkMaxSize(count);
7576
if (values.length < count + 1) {
7677
values = ArrayUtil.grow(values, count + 1);
7778
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ static List<Whitelist> whitelist() {
5050
public static final String[] PARAMETERS = {};
5151

5252
public interface Factory extends ScriptFactory {
53-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
53+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
5454
}
5555

5656
public interface LeafFactory {
@@ -60,8 +60,8 @@ public interface LeafFactory {
6060
private BytesRef[] values = new BytesRef[1];
6161
private int count;
6262

63-
public IpScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
64-
super(params, searchLookup, ctx);
63+
public IpScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
64+
super(fieldName, params, searchLookup, ctx);
6565
}
6666

6767
/**
@@ -94,6 +94,7 @@ public final int count() {
9494
}
9595

9696
protected final void emitValue(String v) {
97+
checkMaxSize(count);
9798
if (values.length < count + 1) {
9899
values = ArrayUtil.grow(values, count + 1);
99100
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,15 @@ static List<Whitelist> whitelist() {
2828
public static final String[] PARAMETERS = {};
2929

3030
public interface Factory extends ScriptFactory {
31-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
31+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
3232
}
3333

3434
public interface LeafFactory {
3535
LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
3636
}
3737

38-
public LongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
39-
super(params, searchLookup, ctx);
38+
public LongScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
39+
super(fieldName, params, searchLookup, ctx);
4040
}
4141

4242
public static class EmitValue {

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java

+24-3
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,15 @@
1717
import java.util.ArrayList;
1818
import java.util.Collections;
1919
import java.util.List;
20+
import java.util.Locale;
2021
import java.util.Map;
2122

2223
public abstract class StringScriptFieldScript extends AbstractScriptFieldScript {
24+
/**
25+
* The maximum number of chars a script should be allowed to emit.
26+
*/
27+
public static final long MAX_CHARS = 1024 * 1024;
28+
2329
public static final ScriptContext<Factory> CONTEXT = newContext("string_script_field", Factory.class);
2430

2531
static List<Whitelist> whitelist() {
@@ -31,17 +37,18 @@ static List<Whitelist> whitelist() {
3137
public static final String[] PARAMETERS = {};
3238

3339
public interface Factory extends ScriptFactory {
34-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
40+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
3541
}
3642

3743
public interface LeafFactory {
3844
StringScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
3945
}
4046

4147
private final List<String> results = new ArrayList<>();
48+
private long chars;
4249

43-
public StringScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
44-
super(params, searchLookup, ctx);
50+
public StringScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
51+
super(fieldName, params, searchLookup, ctx);
4552
}
4653

4754
/**
@@ -52,12 +59,26 @@ public StringScriptFieldScript(Map<String, Object> params, SearchLookup searchLo
5259
*/
5360
public final List<String> resultsForDoc(int docId) {
5461
results.clear();
62+
chars = 0;
5563
setDocument(docId);
5664
execute();
5765
return results;
5866
}
5967

6068
protected final void emitValue(String v) {
69+
checkMaxSize(results.size());
70+
chars += v.length();
71+
if (chars > MAX_CHARS) {
72+
throw new IllegalArgumentException(
73+
String.format(
74+
Locale.ROOT,
75+
"Runtime field [%s] is emitting [%s] characters while the maximum number of values allowed is [%s]",
76+
fieldName,
77+
chars,
78+
MAX_CHARS
79+
)
80+
);
81+
}
6182
results.add(v);
6283
}
6384

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptBooleanMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public ScriptBooleanFieldData.Builder fielddataBuilder(String fullyQualifiedInde
7272
}
7373

7474
private BooleanScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
75-
return scriptFactory.newFactory(script.getParams(), searchLookup);
75+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
7676
}
7777

7878
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ public ScriptDateFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa
8585
}
8686

8787
private DateScriptFieldScript.LeafFactory leafFactory(SearchLookup lookup) {
88-
return scriptFactory.newFactory(script.getParams(), lookup, dateTimeFormatter);
88+
return scriptFactory.newFactory(name(), script.getParams(), lookup, dateTimeFormatter);
8989
}
9090

9191
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public ScriptDoubleFieldData.Builder fielddataBuilder(String fullyQualifiedIndex
6464
}
6565

6666
private DoubleScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
67-
return scriptFactory.newFactory(script.getParams(), searchLookup);
67+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
6868
}
6969

7070
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public ScriptIpFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName
8080
}
8181

8282
private IpScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
83-
return scriptFactory.newFactory(script.getParams(), searchLookup);
83+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
8484
}
8585

8686
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public ScriptStringFieldData.Builder fielddataBuilder(String fullyQualifiedIndex
6868
}
6969

7070
private StringScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
71-
return scriptFactory.newFactory(script.getParams(), searchLookup);
71+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
7272
}
7373

7474
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public ScriptLongFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa
6464
}
6565

6666
private LongScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
67-
return scriptFactory.newFactory(script.getParams(), searchLookup);
67+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
6868
}
6969

7070
@Override

‎x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScriptTests.java

+36-1
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,22 @@
66

77
package org.elasticsearch.xpack.runtimefields;
88

9+
import org.apache.lucene.document.StoredField;
10+
import org.apache.lucene.index.DirectoryReader;
11+
import org.apache.lucene.index.RandomIndexWriter;
12+
import org.apache.lucene.store.Directory;
13+
import org.apache.lucene.util.BytesRef;
14+
import org.elasticsearch.index.mapper.MapperService;
915
import org.elasticsearch.script.ScriptContext;
16+
import org.elasticsearch.search.lookup.SearchLookup;
17+
18+
import java.io.IOException;
19+
20+
import static org.mockito.Mockito.mock;
1021

1122
public class BooleanScriptFieldScriptTests extends ScriptFieldScriptTestCase<BooleanScriptFieldScript.Factory> {
12-
public static final BooleanScriptFieldScript.Factory DUMMY = (params, lookup) -> ctx -> new BooleanScriptFieldScript(
23+
public static final BooleanScriptFieldScript.Factory DUMMY = (fieldName, params, lookup) -> ctx -> new BooleanScriptFieldScript(
24+
fieldName,
1325
params,
1426
lookup,
1527
ctx
@@ -29,4 +41,27 @@ protected ScriptContext<BooleanScriptFieldScript.Factory> context() {
2941
protected BooleanScriptFieldScript.Factory dummyScript() {
3042
return DUMMY;
3143
}
44+
45+
public void testTooManyValues() throws IOException {
46+
try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
47+
iw.addDocument(org.elasticsearch.common.collect.List.of(new StoredField("_source", new BytesRef("{}"))));
48+
try (DirectoryReader reader = iw.getReader()) {
49+
BooleanScriptFieldScript script = new BooleanScriptFieldScript(
50+
"test",
51+
org.elasticsearch.common.collect.Map.of(),
52+
new SearchLookup(mock(MapperService.class), (ft, lookup) -> null, null),
53+
reader.leaves().get(0)
54+
) {
55+
@Override
56+
public void execute() {
57+
for (int i = 0; i <= AbstractScriptFieldScript.MAX_VALUES * 1000; i++) {
58+
emitValue(i % 2 == 0);
59+
}
60+
}
61+
};
62+
// There isn't a limit to the number of values so this won't throw
63+
script.execute();
64+
}
65+
}
66+
}
3267
}

0 commit comments

Comments
 (0)