Skip to content

Commit 4bf3068

Browse files
authored
Adjust DotExpandingXContentParser behaviour (#83313)
With #79922 we have introduced a parser that expands dots in fields names on the fly, so that the expansion no longer needs to be handled by consumers. The currentName method of such parser was not returning all the time the expected name, compared to the corresponding parser that receives as input the document with expanded fields. This commit expands testing and addresses the issues that were found.
1 parent 6152ffd commit 4bf3068

File tree

4 files changed

+137
-48
lines changed

4 files changed

+137
-48
lines changed

libs/x-content/src/main/java/org/elasticsearch/xcontent/DotExpandingXContentParser.java

Lines changed: 57 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -89,11 +89,11 @@ private static String[] splitAndValidatePath(String fullFieldPath) {
8989
for (String part : parts) {
9090
// check if the field name contains only whitespace
9191
if (part.isEmpty()) {
92-
throw new IllegalArgumentException("object field cannot contain only whitespace: ['" + fullFieldPath + "']");
92+
throw new IllegalArgumentException("field name cannot contain only whitespace: ['" + fullFieldPath + "']");
9393
}
9494
if (part.isBlank()) {
9595
throw new IllegalArgumentException(
96-
"object field starting or ending with a [.] makes object resolution ambiguous: [" + fullFieldPath + "]"
96+
"field name starting or ending with a [.] makes object resolution ambiguous: [" + fullFieldPath + "]"
9797
);
9898
}
9999
}
@@ -110,16 +110,17 @@ public static XContentParser expandDots(XContentParser in) throws IOException {
110110
}
111111

112112
private enum State {
113-
PRE,
114-
DURING,
115-
POST
113+
EXPANDING_START_OBJECT,
114+
PARSING_ORIGINAL_CONTENT,
115+
ENDING_EXPANDED_OBJECT
116116
}
117117

118118
final String[] subPaths;
119119
final XContentParser subparser;
120120

121-
int level = 0;
122-
private State state = State.PRE;
121+
private int expandedTokens = 0;
122+
private int innerLevel = -1;
123+
private State state = State.EXPANDING_START_OBJECT;
123124

124125
private DotExpandingXContentParser(XContentParser subparser, XContentParser root, String[] subPaths) {
125126
super(root);
@@ -129,88 +130,102 @@ private DotExpandingXContentParser(XContentParser subparser, XContentParser root
129130

130131
@Override
131132
public Token nextToken() throws IOException {
132-
if (state == State.PRE) {
133-
level++;
134-
if (level == subPaths.length * 2 - 1) {
135-
state = State.DURING;
136-
return in.currentToken();
133+
if (state == State.EXPANDING_START_OBJECT) {
134+
expandedTokens++;
135+
assert expandedTokens < subPaths.length * 2;
136+
if (expandedTokens == subPaths.length * 2 - 1) {
137+
state = State.PARSING_ORIGINAL_CONTENT;
138+
Token token = in.currentToken();
139+
if (token == Token.START_OBJECT || token == Token.START_ARRAY) {
140+
innerLevel++;
141+
}
142+
return token;
137143
}
138-
if (level % 2 == 0) {
144+
// The expansion consists of adding pairs of START_OBJECT and FIELD_NAME tokens
145+
if (expandedTokens % 2 == 0) {
139146
return Token.FIELD_NAME;
140147
}
141148
return Token.START_OBJECT;
142149
}
143-
if (state == State.DURING) {
150+
if (state == State.PARSING_ORIGINAL_CONTENT) {
144151
Token token = subparser.nextToken();
152+
if (token == Token.START_OBJECT || token == Token.START_ARRAY) {
153+
innerLevel++;
154+
}
155+
if (token == Token.END_OBJECT || token == Token.END_ARRAY) {
156+
innerLevel--;
157+
}
145158
if (token != null) {
146159
return token;
147160
}
148-
state = State.POST;
149-
}
150-
assert state == State.POST;
151-
if (level >= 1) {
152-
level -= 2;
161+
state = State.ENDING_EXPANDED_OBJECT;
153162
}
154-
return level < 0 ? null : Token.END_OBJECT;
163+
assert expandedTokens % 2 == 1;
164+
expandedTokens -= 2;
165+
return expandedTokens < 0 ? null : Token.END_OBJECT;
155166
}
156167

157168
@Override
158169
public Token currentToken() {
159-
if (state == State.PRE) {
160-
return level % 2 == 1 ? Token.START_OBJECT : Token.FIELD_NAME;
161-
}
162-
if (state == State.POST) {
163-
if (level > 1) {
164-
return Token.END_OBJECT;
165-
}
166-
}
167-
return in.currentToken();
170+
return switch (state) {
171+
case EXPANDING_START_OBJECT -> expandedTokens % 2 == 1 ? Token.START_OBJECT : Token.FIELD_NAME;
172+
case ENDING_EXPANDED_OBJECT -> Token.END_OBJECT;
173+
case PARSING_ORIGINAL_CONTENT -> in.currentToken();
174+
};
168175
}
169176

170177
@Override
171178
public String currentName() throws IOException {
172-
if (state == State.DURING) {
173-
return in.currentName();
174-
}
175-
if (state == State.POST) {
176-
if (level <= 1) {
179+
if (state == State.PARSING_ORIGINAL_CONTENT) {
180+
assert expandedTokens == subPaths.length * 2 - 1;
181+
// whenever we are parsing some inner object/array we can easily delegate to the inner parser
182+
// e.g. field.with.dots: { obj:{ parsing here } }
183+
if (innerLevel > 0) {
184+
return in.currentName();
185+
}
186+
Token token = currentToken();
187+
// if we are parsing the outer object/array, only at the start object/array we need to return
188+
// e.g. dots instead of field.with.dots otherwise we can simply delegate to the inner parser
189+
// which will do the right thing
190+
if (innerLevel == 0 && token != Token.START_OBJECT && token != Token.START_ARRAY) {
177191
return in.currentName();
178192
}
179-
throw new IllegalStateException("Can't get current name during END_OBJECT");
193+
// note that innerLevel can be -1 if there are no inner object/array e.g. field.with.dots: value
194+
// as well as while there is and we are parsing their END_OBJECT or END_ARRAY
180195
}
181-
return subPaths[level / 2];
196+
return subPaths[expandedTokens / 2];
182197
}
183198

184199
@Override
185200
public void skipChildren() throws IOException {
186-
if (state == State.PRE) {
201+
if (state == State.EXPANDING_START_OBJECT) {
187202
in.skipChildren();
188-
state = State.POST;
203+
state = State.ENDING_EXPANDED_OBJECT;
189204
}
190-
if (state == State.DURING) {
205+
if (state == State.PARSING_ORIGINAL_CONTENT) {
191206
subparser.skipChildren();
192207
}
193208
}
194209

195210
@Override
196211
public String textOrNull() throws IOException {
197-
if (state == State.PRE) {
212+
if (state == State.EXPANDING_START_OBJECT) {
198213
throw new IllegalStateException("Can't get text on a " + currentToken() + " at " + getTokenLocation());
199214
}
200215
return super.textOrNull();
201216
}
202217

203218
@Override
204219
public Number numberValue() throws IOException {
205-
if (state == State.PRE) {
220+
if (state == State.EXPANDING_START_OBJECT) {
206221
throw new IllegalStateException("Can't get numeric value on a " + currentToken() + " at " + getTokenLocation());
207222
}
208223
return super.numberValue();
209224
}
210225

211226
@Override
212227
public boolean booleanValue() throws IOException {
213-
if (state == State.PRE) {
228+
if (state == State.EXPANDING_START_OBJECT) {
214229
throw new IllegalStateException("Can't get boolean value on a " + currentToken() + " at " + getTokenLocation());
215230
}
216231
return super.booleanValue();

libs/x-content/src/test/java/org/elasticsearch/xcontent/DotExpandingXContentParserTests.java

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,23 @@
1616

1717
public class DotExpandingXContentParserTests extends ESTestCase {
1818

19-
private void assertXContentMatches(String expected, String actual) throws IOException {
20-
XContentParser inputParser = createParser(JsonXContent.jsonXContent, actual);
19+
private void assertXContentMatches(String dotsExpanded, String withDots) throws IOException {
20+
XContentParser inputParser = createParser(JsonXContent.jsonXContent, withDots);
2121
XContentParser expandedParser = DotExpandingXContentParser.expandDots(inputParser);
2222

2323
XContentBuilder actualOutput = XContentBuilder.builder(JsonXContent.jsonXContent).copyCurrentStructure(expandedParser);
24-
assertEquals(expected, Strings.toString(actualOutput));
24+
assertEquals(dotsExpanded, Strings.toString(actualOutput));
25+
26+
XContentParser expectedParser = createParser(JsonXContent.jsonXContent, dotsExpanded);
27+
XContentParser actualParser = DotExpandingXContentParser.expandDots(createParser(JsonXContent.jsonXContent, withDots));
28+
XContentParser.Token currentToken;
29+
while ((currentToken = actualParser.nextToken()) != null) {
30+
assertEquals(currentToken, expectedParser.nextToken());
31+
assertEquals(expectedParser.currentToken(), actualParser.currentToken());
32+
assertEquals(actualParser.currentToken().name(), expectedParser.currentName(), actualParser.currentName());
33+
34+
}
35+
assertNull(expectedParser.nextToken());
2536
}
2637

2738
public void testEmbeddedObject() throws IOException {
@@ -33,7 +44,16 @@ public void testEmbeddedObject() throws IOException {
3344
""");
3445
}
3546

36-
public void testEmbeddedArray() throws IOException {
47+
public void testEmbeddedObjects() throws IOException {
48+
49+
assertXContentMatches("""
50+
{"test":{"with":{"dots":{"obj":{"field":"value","array":["value",{"field":"value"}]}}}},"nodots":"value2"}\
51+
""", """
52+
{"test.with.dots":{"obj":{"field":"value","array":["value",{"field":"value"}]}},"nodots":"value2"}\
53+
""");
54+
}
55+
56+
public void testEmbeddedArrayOfValues() throws IOException {
3757

3858
assertXContentMatches("""
3959
{"test":{"with":{"dots":["field","value"]}},"nodots":"value2"}\
@@ -43,6 +63,26 @@ public void testEmbeddedArray() throws IOException {
4363

4464
}
4565

66+
public void testEmbeddedArrayOfObjects() throws IOException {
67+
68+
assertXContentMatches("""
69+
{"test":{"with":{"dots":[{"field":"value"},{"field":"value"}]}},"nodots":"value2"}\
70+
""", """
71+
{"test.with.dots":[{"field":"value"},{"field":"value"}],"nodots":"value2"}\
72+
""");
73+
74+
}
75+
76+
public void testEmbeddedArrayMixedContent() throws IOException {
77+
78+
assertXContentMatches("""
79+
{"test":{"with":{"dots":["value",{"field":"value"}]}},"nodots":"value2"}\
80+
""", """
81+
{"test.with.dots":["value",{"field":"value"}],"nodots":"value2"}\
82+
""");
83+
84+
}
85+
4686
public void testEmbeddedValue() throws IOException {
4787

4888
assertXContentMatches("""
@@ -85,6 +125,40 @@ public void testSkipChildren() throws IOException {
85125
assertNull(parser.nextToken());
86126
}
87127

128+
public void testSkipChildrenWithinInnerObject() throws IOException {
129+
XContentParser parser = DotExpandingXContentParser.expandDots(createParser(JsonXContent.jsonXContent, """
130+
{ "test.with.dots" : {"obj" : {"field":"value"}}, "nodots" : "value2" }"""));
131+
132+
parser.nextToken(); // start object
133+
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
134+
assertEquals("test", parser.currentName());
135+
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
136+
assertEquals(XContentParser.Token.START_OBJECT, parser.currentToken());
137+
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
138+
assertEquals("with", parser.currentName());
139+
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
140+
assertEquals(XContentParser.Token.START_OBJECT, parser.currentToken());
141+
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
142+
assertEquals("dots", parser.currentName());
143+
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
144+
assertEquals(XContentParser.Token.START_OBJECT, parser.currentToken());
145+
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
146+
assertEquals("obj", parser.currentName());
147+
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
148+
assertEquals(XContentParser.Token.START_OBJECT, parser.currentToken());
149+
parser.skipChildren();
150+
assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
151+
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
152+
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
153+
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
154+
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
155+
assertEquals("nodots", parser.currentName());
156+
assertEquals(XContentParser.Token.VALUE_STRING, parser.nextToken());
157+
assertEquals("value2", parser.text());
158+
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
159+
assertNull(parser.nextToken());
160+
}
161+
88162
public void testNestedExpansions() throws IOException {
89163
assertXContentMatches("""
90164
{"first":{"dot":{"second":{"dot":"value"},"third":"value"}},"nodots":"value"}\

server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -440,7 +440,7 @@ public void testDynamicRuntimeObjectFields() {
440440
() -> client().prepareIndex("test").setSource("obj.runtime", "value").get()
441441
);
442442
assertEquals(
443-
"object mapping for [obj.runtime] tried to parse field [obj.runtime] as object, but found a concrete value",
443+
"object mapping for [obj.runtime] tried to parse field [runtime] as object, but found a concrete value",
444444
exception.getMessage()
445445
);
446446

server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1787,7 +1787,7 @@ public void testDynamicFieldsStartingAndEndingWithDot() throws Exception {
17871787
{"top..foo.":{"a":1}}
17881788
""")));
17891789

1790-
assertThat(e.getCause().getMessage(), containsString("object field cannot contain only whitespace: ['top..foo.']"));
1790+
assertThat(e.getCause().getMessage(), containsString("field name cannot contain only whitespace: ['top..foo.']"));
17911791
}
17921792

17931793
public void testDynamicFieldsEmptyName() throws Exception {

0 commit comments

Comments
 (0)