Skip to content

Commit 8ef7438

Browse files
sneivandttalevy
sneivandt
authored and
talevy
committed
Accept ingest simulate params as ints or strings (#23885)
* Allow ingest simulate to parse _id, _index, _type, _routing and _parent as either string or int (#23823) * Generate data that includes Integer and String type fields for testing document parsing.
1 parent 4f5ce55 commit 8ef7438

File tree

4 files changed

+98
-18
lines changed

4 files changed

+98
-18
lines changed

‎core/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -174,16 +174,24 @@ static Parsed parse(Map<String, Object> config, boolean verbose, PipelineStore p
174174
}
175175

176176
private static List<IngestDocument> parseDocs(Map<String, Object> config) {
177-
List<Map<String, Object>> docs = ConfigurationUtils.readList(null, null, config, Fields.DOCS);
177+
List<Map<String, Object>> docs =
178+
ConfigurationUtils.readList(null, null, config, Fields.DOCS);
178179
List<IngestDocument> ingestDocumentList = new ArrayList<>();
179180
for (Map<String, Object> dataMap : docs) {
180-
Map<String, Object> document = ConfigurationUtils.readMap(null, null, dataMap, Fields.SOURCE);
181-
IngestDocument ingestDocument = new IngestDocument(ConfigurationUtils.readStringProperty(null, null, dataMap, MetaData.INDEX.getFieldName(), "_index"),
182-
ConfigurationUtils.readStringProperty(null, null, dataMap, MetaData.TYPE.getFieldName(), "_type"),
183-
ConfigurationUtils.readStringProperty(null, null, dataMap, MetaData.ID.getFieldName(), "_id"),
184-
ConfigurationUtils.readOptionalStringProperty(null, null, dataMap, MetaData.ROUTING.getFieldName()),
185-
ConfigurationUtils.readOptionalStringProperty(null, null, dataMap, MetaData.PARENT.getFieldName()),
186-
document);
181+
Map<String, Object> document = ConfigurationUtils.readMap(null, null,
182+
dataMap, Fields.SOURCE);
183+
String index = ConfigurationUtils.readStringOrIntProperty(null, null,
184+
dataMap, MetaData.INDEX.getFieldName(), "_index");
185+
String type = ConfigurationUtils.readStringOrIntProperty(null, null,
186+
dataMap, MetaData.TYPE.getFieldName(), "_type");
187+
String id = ConfigurationUtils.readStringOrIntProperty(null, null,
188+
dataMap, MetaData.ID.getFieldName(), "_id");
189+
String routing = ConfigurationUtils.readOptionalStringOrIntProperty(null, null,
190+
dataMap, MetaData.ROUTING.getFieldName());
191+
String parent = ConfigurationUtils.readOptionalStringOrIntProperty(null, null,
192+
dataMap, MetaData.PARENT.getFieldName());
193+
IngestDocument ingestDocument =
194+
new IngestDocument(index, type, id, routing, parent, document);
187195
ingestDocumentList.add(ingestDocument);
188196
}
189197
return ingestDocumentList;

‎core/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,52 @@ private static String readString(String processorType, String processorTag, Stri
9292
value.getClass().getName() + "]");
9393
}
9494

95+
/**
96+
* Returns and removes the specified property from the specified configuration map.
97+
*
98+
* If the property value isn't of type string or int a {@link ElasticsearchParseException} is thrown.
99+
* If the property is missing and no default value has been specified a {@link ElasticsearchParseException} is thrown
100+
*/
101+
public static String readStringOrIntProperty(String processorType, String processorTag,
102+
Map<String, Object> configuration, String propertyName, String defaultValue) {
103+
Object value = configuration.remove(propertyName);
104+
if (value == null && defaultValue != null) {
105+
return defaultValue;
106+
} else if (value == null) {
107+
throw newConfigurationException(processorType, processorTag, propertyName,
108+
"required property is missing");
109+
}
110+
return readStringOrInt(processorType, processorTag, propertyName, value);
111+
}
112+
113+
private static String readStringOrInt(String processorType, String processorTag,
114+
String propertyName, Object value) {
115+
if (value == null) {
116+
return null;
117+
}
118+
if (value instanceof String) {
119+
return (String) value;
120+
} else if (value instanceof Integer) {
121+
return String.valueOf(value);
122+
}
123+
throw newConfigurationException(processorType, processorTag, propertyName,
124+
"property isn't a string or int, but of type [" + value.getClass().getName() + "]");
125+
}
126+
127+
/**
128+
* Returns and removes the specified property from the specified configuration map.
129+
*
130+
* If the property value isn't of type string or int a {@link ElasticsearchParseException} is thrown.
131+
*/
132+
public static String readOptionalStringOrIntProperty(String processorType, String processorTag,
133+
Map<String, Object> configuration, String propertyName) {
134+
Object value = configuration.remove(propertyName);
135+
if (value == null) {
136+
return null;
137+
}
138+
return readStringOrInt(processorType, processorTag, propertyName, value);
139+
}
140+
95141
public static Boolean readBooleanProperty(String processorType, String processorTag, Map<String, Object> configuration,
96142
String propertyName, boolean defaultValue) {
97143
Object value = configuration.remove(propertyName);

‎core/src/test/java/org/elasticsearch/action/ingest/SimulatePipelineRequestParsingTests.java

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import java.io.IOException;
2323
import java.util.ArrayList;
24+
import java.util.Arrays;
2425
import java.util.Collections;
2526
import java.util.HashMap;
2627
import java.util.Iterator;
@@ -40,6 +41,8 @@
4041
import static org.elasticsearch.action.ingest.SimulatePipelineRequest.SIMULATED_PIPELINE_ID;
4142
import static org.elasticsearch.ingest.IngestDocument.MetaData.ID;
4243
import static org.elasticsearch.ingest.IngestDocument.MetaData.INDEX;
44+
import static org.elasticsearch.ingest.IngestDocument.MetaData.PARENT;
45+
import static org.elasticsearch.ingest.IngestDocument.MetaData.ROUTING;
4346
import static org.elasticsearch.ingest.IngestDocument.MetaData.TYPE;
4447
import static org.hamcrest.Matchers.equalTo;
4548
import static org.hamcrest.Matchers.nullValue;
@@ -116,20 +119,24 @@ public void testParseWithProvidedPipeline() throws Exception {
116119
requestContent.put(Fields.DOCS, docs);
117120
for (int i = 0; i < numDocs; i++) {
118121
Map<String, Object> doc = new HashMap<>();
119-
String index = randomAlphaOfLengthBetween(1, 10);
120-
String type = randomAlphaOfLengthBetween(1, 10);
121-
String id = randomAlphaOfLengthBetween(1, 10);
122-
doc.put(INDEX.getFieldName(), index);
123-
doc.put(TYPE.getFieldName(), type);
124-
doc.put(ID.getFieldName(), id);
122+
Map<String, Object> expectedDoc = new HashMap<>();
123+
List<IngestDocument.MetaData> fields = Arrays.asList(INDEX, TYPE, ID, ROUTING, PARENT);
124+
for(IngestDocument.MetaData field : fields) {
125+
if(randomBoolean()) {
126+
String value = randomAlphaOfLengthBetween(1, 10);
127+
doc.put(field.getFieldName(), value);
128+
expectedDoc.put(field.getFieldName(), value);
129+
}
130+
else {
131+
Integer value = randomIntBetween(1, 1000000);
132+
doc.put(field.getFieldName(), value);
133+
expectedDoc.put(field.getFieldName(), String.valueOf(value));
134+
}
135+
}
125136
String fieldName = randomAlphaOfLengthBetween(1, 10);
126137
String fieldValue = randomAlphaOfLengthBetween(1, 10);
127138
doc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
128139
docs.add(doc);
129-
Map<String, Object> expectedDoc = new HashMap<>();
130-
expectedDoc.put(INDEX.getFieldName(), index);
131-
expectedDoc.put(TYPE.getFieldName(), type);
132-
expectedDoc.put(ID.getFieldName(), id);
133140
expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
134141
expectedDocs.add(expectedDoc);
135142
}
@@ -172,6 +179,8 @@ public void testParseWithProvidedPipeline() throws Exception {
172179
assertThat(metadataMap.get(INDEX), equalTo(expectedDocument.get(INDEX.getFieldName())));
173180
assertThat(metadataMap.get(TYPE), equalTo(expectedDocument.get(TYPE.getFieldName())));
174181
assertThat(metadataMap.get(ID), equalTo(expectedDocument.get(ID.getFieldName())));
182+
assertThat(metadataMap.get(ROUTING), equalTo(expectedDocument.get(ROUTING.getFieldName())));
183+
assertThat(metadataMap.get(PARENT), equalTo(expectedDocument.get(PARENT.getFieldName())));
175184
assertThat(ingestDocument.getSourceAndMetadata(), equalTo(expectedDocument.get(Fields.SOURCE)));
176185
}
177186

‎core/src/test/java/org/elasticsearch/ingest/ConfigurationUtilsTests.java

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ public void setConfig() {
5454
Map<String, Object> fizz = new HashMap<>();
5555
fizz.put("buzz", "hello world");
5656
config.put("fizz", fizz);
57+
config.put("num", 1);
5758
}
5859

5960
public void testReadStringProperty() {
@@ -93,6 +94,22 @@ public void testOptional_InvalidType() {
9394
assertThat(val, equalTo(Collections.singletonList(2)));
9495
}
9596

97+
public void testReadStringOrIntProperty() {
98+
String val1 = ConfigurationUtils.readStringOrIntProperty(null, null, config, "foo", null);
99+
String val2 = ConfigurationUtils.readStringOrIntProperty(null, null, config, "num", null);
100+
assertThat(val1, equalTo("bar"));
101+
assertThat(val2, equalTo("1"));
102+
}
103+
104+
public void testReadStringOrIntPropertyInvalidType() {
105+
try {
106+
ConfigurationUtils.readStringOrIntProperty(null, null, config, "arr", null);
107+
} catch (ElasticsearchParseException e) {
108+
assertThat(e.getMessage(), equalTo(
109+
"[arr] property isn't a string or int, but of type [java.util.Arrays$ArrayList]"));
110+
}
111+
}
112+
96113
public void testReadProcessors() throws Exception {
97114
Processor processor = mock(Processor.class);
98115
Map<String, Processor.Factory> registry =

0 commit comments

Comments
 (0)