Skip to content

Commit 7f542e0

Browse files
nik9000nik9000
authored
Stop runtime script from emitting too many values (#61938)
This prevent `keyword` valued runtime scripts from emitting too many values or values that take up too much space. Without this you can put allocate a ton of memory with the script by sticking it into a tight loop. Painless has some protections against this but: 1. I don't want to rely on them out of sheer paranoia 2. They don't really kick in when the script uses callbacks like we do anyway. Relates to #59332
1 parent c8707ea commit 7f542e0

27 files changed

+408
-47
lines changed

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java

+3-2
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ public abstract class AbstractLongScriptFieldScript extends AbstractScriptFieldS
1919
private long[] values = new long[1];
2020
private int count;
2121

22-
public AbstractLongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
23-
super(params, searchLookup, ctx);
22+
public AbstractLongScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
23+
super(fieldName, params, searchLookup, ctx);
2424
}
2525

2626
/**
@@ -50,6 +50,7 @@ public final int count() {
5050
}
5151

5252
protected final void emitValue(long v) {
53+
checkMaxSize(count);
5354
if (values.length < count + 1) {
5455
values = ArrayUtil.grow(values, count + 1);
5556
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java

+27-1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
import org.elasticsearch.search.lookup.SourceLookup;
1818

1919
import java.util.HashMap;
20+
import java.util.Locale;
2021
import java.util.Map;
2122
import java.util.function.Function;
2223

@@ -27,6 +28,11 @@
2728
* {@link AggregationScript} but hopefully with less historical baggage.
2829
*/
2930
public abstract class AbstractScriptFieldScript {
31+
/**
32+
* The maximum number of values a script should be allowed to emit.
33+
*/
34+
static final int MAX_VALUES = 100;
35+
3036
public static <F> ScriptContext<F> newContext(String name, Class<F> factoryClass) {
3137
return new ScriptContext<F>(
3238
name + "_script_field",
@@ -54,10 +60,12 @@ public static <F> ScriptContext<F> newContext(String name, Class<F> factoryClass
5460
value -> ((SourceLookup) value).loadSourceIfNeeded()
5561
);
5662

63+
protected final String fieldName;
5764
private final Map<String, Object> params;
5865
private final LeafSearchLookup leafSearchLookup;
5966

60-
public AbstractScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
67+
public AbstractScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
68+
this.fieldName = fieldName;
6169
this.leafSearchLookup = searchLookup.getLeafSearchLookup(ctx);
6270
params = new HashMap<>(params);
6371
params.put("_source", leafSearchLookup.source());
@@ -93,5 +101,23 @@ public final Map<String, ScriptDocValues<?>> getDoc() {
93101
return leafSearchLookup.doc();
94102
}
95103

104+
/**
105+
* Check if the we can add another value to the list of values.
106+
* @param currentSize the current size of the list
107+
*/
108+
protected final void checkMaxSize(int currentSize) {
109+
if (currentSize >= MAX_VALUES) {
110+
throw new IllegalArgumentException(
111+
String.format(
112+
Locale.ROOT,
113+
"Runtime field [%s] is emitting [%s] values while the maximum number of values allowed is [%s]",
114+
fieldName,
115+
currentSize + 1,
116+
MAX_VALUES
117+
)
118+
);
119+
}
120+
}
121+
96122
public abstract void execute();
97123
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScript.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ static List<Whitelist> whitelist() {
2828
public static final String[] PARAMETERS = {};
2929

3030
public interface Factory extends ScriptFactory {
31-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
31+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
3232
}
3333

3434
public interface LeafFactory {
@@ -38,8 +38,8 @@ public interface LeafFactory {
3838
private int trues;
3939
private int falses;
4040

41-
public BooleanScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
42-
super(params, searchLookup, ctx);
41+
public BooleanScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
42+
super(fieldName, params, searchLookup, ctx);
4343
}
4444

4545
/**

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java

+9-3
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ static List<Whitelist> whitelist() {
3030
public static final String[] PARAMETERS = {};
3131

3232
public interface Factory extends ScriptFactory {
33-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter);
33+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter);
3434
}
3535

3636
public interface LeafFactory {
@@ -39,8 +39,14 @@ public interface LeafFactory {
3939

4040
private final DateFormatter formatter;
4141

42-
public DateScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter, LeafReaderContext ctx) {
43-
super(params, searchLookup, ctx);
42+
public DateScriptFieldScript(
43+
String fieldName,
44+
Map<String, Object> params,
45+
SearchLookup searchLookup,
46+
DateFormatter formatter,
47+
LeafReaderContext ctx
48+
) {
49+
super(fieldName, params, searchLookup, ctx);
4450
this.formatter = formatter;
4551
}
4652

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ static List<Whitelist> whitelist() {
2828
public static final String[] PARAMETERS = {};
2929

3030
public interface Factory extends ScriptFactory {
31-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
31+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
3232
}
3333

3434
public interface LeafFactory {
@@ -38,8 +38,8 @@ public interface LeafFactory {
3838
private double[] values = new double[1];
3939
private int count;
4040

41-
public DoubleScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
42-
super(params, searchLookup, ctx);
41+
public DoubleScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
42+
super(fieldName, params, searchLookup, ctx);
4343
}
4444

4545
/**
@@ -69,6 +69,7 @@ public final int count() {
6969
}
7070

7171
protected final void emitValue(double v) {
72+
checkMaxSize(count);
7273
if (values.length < count + 1) {
7374
values = ArrayUtil.grow(values, count + 1);
7475
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java

+4-3
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ static List<Whitelist> whitelist() {
4949
public static final String[] PARAMETERS = {};
5050

5151
public interface Factory extends ScriptFactory {
52-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
52+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
5353
}
5454

5555
public interface LeafFactory {
@@ -59,8 +59,8 @@ public interface LeafFactory {
5959
private BytesRef[] values = new BytesRef[1];
6060
private int count;
6161

62-
public IpScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
63-
super(params, searchLookup, ctx);
62+
public IpScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
63+
super(fieldName, params, searchLookup, ctx);
6464
}
6565

6666
/**
@@ -93,6 +93,7 @@ public final int count() {
9393
}
9494

9595
protected final void emitValue(String v) {
96+
checkMaxSize(count);
9697
if (values.length < count + 1) {
9798
values = ArrayUtil.grow(values, count + 1);
9899
}

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java

+3-3
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@ static List<Whitelist> whitelist() {
2727
public static final String[] PARAMETERS = {};
2828

2929
public interface Factory extends ScriptFactory {
30-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
30+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
3131
}
3232

3333
public interface LeafFactory {
3434
LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
3535
}
3636

37-
public LongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
38-
super(params, searchLookup, ctx);
37+
public LongScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
38+
super(fieldName, params, searchLookup, ctx);
3939
}
4040

4141
public static class EmitValue {

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java

+24-3
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,15 @@
1616
import java.io.IOException;
1717
import java.util.ArrayList;
1818
import java.util.List;
19+
import java.util.Locale;
1920
import java.util.Map;
2021

2122
public abstract class StringScriptFieldScript extends AbstractScriptFieldScript {
23+
/**
24+
* The maximum number of chars a script should be allowed to emit.
25+
*/
26+
public static final long MAX_CHARS = 1024 * 1024;
27+
2228
public static final ScriptContext<Factory> CONTEXT = newContext("string_script_field", Factory.class);
2329

2430
static List<Whitelist> whitelist() {
@@ -28,17 +34,18 @@ static List<Whitelist> whitelist() {
2834
public static final String[] PARAMETERS = {};
2935

3036
public interface Factory extends ScriptFactory {
31-
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
37+
LeafFactory newFactory(String fieldName, Map<String, Object> params, SearchLookup searchLookup);
3238
}
3339

3440
public interface LeafFactory {
3541
StringScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
3642
}
3743

3844
private final List<String> results = new ArrayList<>();
45+
private long chars;
3946

40-
public StringScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
41-
super(params, searchLookup, ctx);
47+
public StringScriptFieldScript(String fieldName, Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
48+
super(fieldName, params, searchLookup, ctx);
4249
}
4350

4451
/**
@@ -49,12 +56,26 @@ public StringScriptFieldScript(Map<String, Object> params, SearchLookup searchLo
4956
*/
5057
public final List<String> resultsForDoc(int docId) {
5158
results.clear();
59+
chars = 0;
5260
setDocument(docId);
5361
execute();
5462
return results;
5563
}
5664

5765
protected final void emitValue(String v) {
66+
checkMaxSize(results.size());
67+
chars += v.length();
68+
if (chars > MAX_CHARS) {
69+
throw new IllegalArgumentException(
70+
String.format(
71+
Locale.ROOT,
72+
"Runtime field [%s] is emitting [%s] characters while the maximum number of values allowed is [%s]",
73+
fieldName,
74+
chars,
75+
MAX_CHARS
76+
)
77+
);
78+
}
5879
results.add(v);
5980
}
6081

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptBooleanMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ public ScriptBooleanFieldData.Builder fielddataBuilder(String fullyQualifiedInde
7272
}
7373

7474
private BooleanScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
75-
return scriptFactory.newFactory(script.getParams(), searchLookup);
75+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
7676
}
7777

7878
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ public ScriptDateFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa
8585
}
8686

8787
private DateScriptFieldScript.LeafFactory leafFactory(SearchLookup lookup) {
88-
return scriptFactory.newFactory(script.getParams(), lookup, dateTimeFormatter);
88+
return scriptFactory.newFactory(name(), script.getParams(), lookup, dateTimeFormatter);
8989
}
9090

9191
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDoubleMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public ScriptDoubleFieldData.Builder fielddataBuilder(String fullyQualifiedIndex
6464
}
6565

6666
private DoubleScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
67-
return scriptFactory.newFactory(script.getParams(), searchLookup);
67+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
6868
}
6969

7070
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public ScriptIpFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName
8080
}
8181

8282
private IpScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
83-
return scriptFactory.newFactory(script.getParams(), searchLookup);
83+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
8484
}
8585

8686
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ public ScriptStringFieldData.Builder fielddataBuilder(String fullyQualifiedIndex
6868
}
6969

7070
private StringScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
71-
return scriptFactory.newFactory(script.getParams(), searchLookup);
71+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
7272
}
7373

7474
@Override

‎x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldType.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ public ScriptLongFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa
6464
}
6565

6666
private LongScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) {
67-
return scriptFactory.newFactory(script.getParams(), searchLookup);
67+
return scriptFactory.newFactory(name(), script.getParams(), searchLookup);
6868
}
6969

7070
@Override

‎x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/BooleanScriptFieldScriptTests.java

+38-1
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,24 @@
66

77
package org.elasticsearch.xpack.runtimefields;
88

9+
import org.apache.lucene.document.StoredField;
10+
import org.apache.lucene.index.DirectoryReader;
11+
import org.apache.lucene.index.RandomIndexWriter;
12+
import org.apache.lucene.store.Directory;
13+
import org.apache.lucene.util.BytesRef;
14+
import org.elasticsearch.index.mapper.MapperService;
915
import org.elasticsearch.script.ScriptContext;
16+
import org.elasticsearch.search.lookup.SearchLookup;
17+
18+
import java.io.IOException;
19+
import java.util.List;
20+
import java.util.Map;
21+
22+
import static org.mockito.Mockito.mock;
1023

1124
public class BooleanScriptFieldScriptTests extends ScriptFieldScriptTestCase<BooleanScriptFieldScript.Factory> {
12-
public static final BooleanScriptFieldScript.Factory DUMMY = (params, lookup) -> ctx -> new BooleanScriptFieldScript(
25+
public static final BooleanScriptFieldScript.Factory DUMMY = (fieldName, params, lookup) -> ctx -> new BooleanScriptFieldScript(
26+
fieldName,
1327
params,
1428
lookup,
1529
ctx
@@ -29,4 +43,27 @@ protected ScriptContext<BooleanScriptFieldScript.Factory> context() {
2943
protected BooleanScriptFieldScript.Factory dummyScript() {
3044
return DUMMY;
3145
}
46+
47+
public void testTooManyValues() throws IOException {
48+
try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) {
49+
iw.addDocument(List.of(new StoredField("_source", new BytesRef("{}"))));
50+
try (DirectoryReader reader = iw.getReader()) {
51+
BooleanScriptFieldScript script = new BooleanScriptFieldScript(
52+
"test",
53+
Map.of(),
54+
new SearchLookup(mock(MapperService.class), (ft, lookup) -> null),
55+
reader.leaves().get(0)
56+
) {
57+
@Override
58+
public void execute() {
59+
for (int i = 0; i <= AbstractScriptFieldScript.MAX_VALUES * 1000; i++) {
60+
emitValue(i % 2 == 0);
61+
}
62+
}
63+
};
64+
// There isn't a limit to the number of values so this won't throw
65+
script.execute();
66+
}
67+
}
68+
}
3269
}

0 commit comments

Comments
 (0)