Skip to content

Commit 5971781

Browse files
author
Andrew Stucki
committed
Make NetworkDirectionProcessor more robust
1 parent 9cc7152 commit 5971781

File tree

6 files changed

+134
-31
lines changed

6 files changed

+134
-31
lines changed

‎docs/reference/ingest/processors/network-direction.asciidoc

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,18 @@ only the `internal_networks` option must be specified.
2121
| `source_ip` | no | `source.ip` | Field containing the source IP address.
2222
| `destination_ip` | no | `destination.ip` | Field containing the destination IP address.
2323
| `target_field` | no | `network.direction` | Output field for the network direction.
24-
| `internal_networks`| yes | | List of internal networks. Supports IPv4 and
25-
IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below.
24+
| `internal_networks`| no | | List of internal networks. Supports IPv4 and
25+
IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. These may be constructed with <<accessing-template-fields,template snippets>>.
26+
| `internal_networks_field`| no | | A field on the given document to read the `internal_networks` configuration from.
2627
| `ignore_missing` | no | `true` | If `true` and any required fields are missing,
2728
the processor quietly exits without modifying the document.
2829

2930

3031
include::common-options.asciidoc[]
3132
|======
3233

34+
One of either `internal_networks` or `internal_networks_field` must be specified. If `internal_networks_field` is specified, it follows the behavior specified by `ignore_missing`.
35+
3336
[float]
3437
[[supported-named-network-ranges]]
3538
===== Supported named network ranges

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
7979
entry(HtmlStripProcessor.TYPE, new HtmlStripProcessor.Factory()),
8080
entry(CsvProcessor.TYPE, new CsvProcessor.Factory()),
8181
entry(UriPartsProcessor.TYPE, new UriPartsProcessor.Factory()),
82-
entry(NetworkDirectionProcessor.TYPE, new NetworkDirectionProcessor.Factory()),
82+
entry(NetworkDirectionProcessor.TYPE, new NetworkDirectionProcessor.Factory(parameters.scriptService)),
8383
entry(CommunityIdProcessor.TYPE, new CommunityIdProcessor.Factory()),
8484
entry(FingerprintProcessor.TYPE, new FingerprintProcessor.Factory())
8585
);

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java

Lines changed: 58 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,18 @@
1414
import org.elasticsearch.ingest.ConfigurationUtils;
1515
import org.elasticsearch.ingest.IngestDocument;
1616
import org.elasticsearch.ingest.Processor;
17+
import org.elasticsearch.ElasticsearchParseException;
18+
import org.elasticsearch.common.network.InetAddresses;
19+
import org.elasticsearch.script.ScriptService;
20+
import org.elasticsearch.script.TemplateScript;
21+
import org.elasticsearch.common.network.CIDRUtils;
1722

1823
import java.net.InetAddress;
19-
import java.util.Arrays;
24+
import java.util.ArrayList;
2025
import java.util.List;
2126
import java.util.Map;
27+
import java.util.Arrays;
28+
import java.util.stream.Collectors;
2229

2330
import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
2431

@@ -48,7 +55,8 @@ public class NetworkDirectionProcessor extends AbstractProcessor {
4855
private final String sourceIpField;
4956
private final String destinationIpField;
5057
private final String targetField;
51-
private final List<String> internalNetworks;
58+
private final List<TemplateScript.Factory> internalNetworks;
59+
private final String internalNetworksField;
5260
private final boolean ignoreMissing;
5361

5462
NetworkDirectionProcessor(
@@ -57,14 +65,16 @@ public class NetworkDirectionProcessor extends AbstractProcessor {
5765
String sourceIpField,
5866
String destinationIpField,
5967
String targetField,
60-
List<String> internalNetworks,
68+
List<TemplateScript.Factory> internalNetworks,
69+
String internalNetworksField,
6170
boolean ignoreMissing
6271
) {
6372
super(tag, description);
6473
this.sourceIpField = sourceIpField;
6574
this.destinationIpField = destinationIpField;
6675
this.targetField = targetField;
6776
this.internalNetworks = internalNetworks;
77+
this.internalNetworksField = internalNetworksField;
6878
this.ignoreMissing = ignoreMissing;
6979
}
7080

@@ -80,7 +90,7 @@ public String getTargetField() {
8090
return targetField;
8191
}
8292

83-
public List<String> getInternalNetworks() {
93+
public List<TemplateScript.Factory> getInternalNetworks() {
8494
return internalNetworks;
8595
}
8696

@@ -103,8 +113,17 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
103113
return ingestDocument;
104114
}
105115

106-
private String getDirection(IngestDocument d) {
107-
if (internalNetworks == null) {
116+
private String getDirection(IngestDocument d) throws Exception {
117+
List<String> networks = new ArrayList<>();
118+
119+
if (internalNetworksField != null) {
120+
@SuppressWarnings("unchecked")
121+
List<String> stringList = d.getFieldValue(internalNetworksField, networks.getClass(), ignoreMissing);
122+
networks.addAll(stringList);
123+
} else if (internalNetworks != null) {
124+
networks = internalNetworks.stream().map(network -> d.renderTemplate(network)).collect(Collectors.toList());
125+
}
126+
if (networks == null) {
108127
return null;
109128
}
110129

@@ -118,8 +137,8 @@ private String getDirection(IngestDocument d) {
118137
return null;
119138
}
120139

121-
boolean sourceInternal = isInternal(sourceIpAddrString);
122-
boolean destinationInternal = isInternal(destIpAddrString);
140+
boolean sourceInternal = isInternal(networks, sourceIpAddrString);
141+
boolean destinationInternal = isInternal(networks, destIpAddrString);
123142

124143
if (sourceInternal && destinationInternal) {
125144
return DIRECTION_INTERNAL;
@@ -133,8 +152,8 @@ private String getDirection(IngestDocument d) {
133152
return DIRECTION_EXTERNAL;
134153
}
135154

136-
private boolean isInternal(String ip) {
137-
for (String network : internalNetworks) {
155+
private boolean isInternal(List<String> networks, String ip) {
156+
for (String network : networks) {
138157
if (inNetwork(ip, network)) {
139158
return true;
140159
}
@@ -227,31 +246,53 @@ public String getType() {
227246
}
228247

229248
public static final class Factory implements Processor.Factory {
230-
249+
private final ScriptService scriptService;
231250
static final String DEFAULT_SOURCE_IP = "source.ip";
232251
static final String DEFAULT_DEST_IP = "destination.ip";
233252
static final String DEFAULT_TARGET = "network.direction";
234253

254+
public Factory(ScriptService scriptService) {
255+
this.scriptService = scriptService;
256+
}
257+
235258
@Override
236259
public NetworkDirectionProcessor create(
237260
Map<String, Processor.Factory> registry,
238261
String processorTag,
239262
String description,
240263
Map<String, Object> config
241264
) throws Exception {
242-
String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP);
243-
String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP);
244-
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET);
245-
List<String> internalNetworks = ConfigurationUtils.readList(TYPE, processorTag, config, "internal_networks");
246-
boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", true);
265+
final String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP);
266+
final String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP);
267+
final String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET);
268+
final boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", true);
269+
270+
final List<String> internalNetworks = ConfigurationUtils.readOptionalList(TYPE, processorTag, config, "internal_networks");
271+
final String internalNetworksField = ConfigurationUtils.readOptionalStringProperty(
272+
TYPE,
273+
processorTag,
274+
config,
275+
"internal_networks_field"
276+
);
247277

278+
if (internalNetworks == null && internalNetworksField == null) {
279+
throw new ElasticsearchParseException("either [internal_networks] or [internal_networks_field] must be specified");
280+
}
281+
282+
List<TemplateScript.Factory> internalNetworkTemplates = null;
283+
if (internalNetworks != null) {
284+
internalNetworkTemplates = internalNetworks.stream()
285+
.map(n -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "internal_networks", n, scriptService))
286+
.collect(Collectors.toList());
287+
}
248288
return new NetworkDirectionProcessor(
249289
processorTag,
250290
description,
251291
sourceIpField,
252292
destIpField,
253293
targetField,
254-
internalNetworks,
294+
internalNetworkTemplates,
295+
internalNetworksField,
255296
ignoreMissing
256297
);
257298
}

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,12 @@
1010

1111
import org.elasticsearch.ElasticsearchParseException;
1212
import org.elasticsearch.test.ESTestCase;
13+
import org.elasticsearch.ingest.TestTemplateService;
1314
import org.junit.Before;
1415

1516
import java.util.ArrayList;
1617
import java.util.HashMap;
18+
import java.util.Collections;
1719
import java.util.List;
1820
import java.util.Map;
1921

@@ -28,7 +30,7 @@ public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
2830

2931
@Before
3032
public void init() {
31-
factory = new NetworkDirectionProcessor.Factory();
33+
factory = new NetworkDirectionProcessor.Factory(TestTemplateService.instance());
3234
}
3335

3436
public void testCreate() throws Exception {
@@ -52,7 +54,7 @@ public void testCreate() throws Exception {
5254
assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField));
5355
assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField));
5456
assertThat(networkProcessor.getTargetField(), equalTo(targetField));
55-
assertThat(networkProcessor.getInternalNetworks(), equalTo(internalNetworks));
57+
assertThat(networkProcessor.getInternalNetworks().get(0).newInstance(Collections.emptyMap()).execute(), equalTo("10.0.0.0/8"));
5658
assertThat(networkProcessor.getIgnoreMissing(), equalTo(ignoreMissing));
5759
}
5860

@@ -63,7 +65,7 @@ public void testRequiredFields() throws Exception {
6365
factory.create(null, processorTag, null, config);
6466
fail("factory create should have failed");
6567
} catch (ElasticsearchParseException e) {
66-
assertThat(e.getMessage(), equalTo("[internal_networks] required property is missing"));
68+
assertThat(e.getMessage(), equalTo("either [internal_networks] or [internal_networks_field] must be specified"));
6769
}
6870
}
6971

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java

Lines changed: 37 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,14 @@
1010

1111
import org.elasticsearch.ingest.IngestDocument;
1212
import org.elasticsearch.test.ESTestCase;
13+
import org.elasticsearch.ingest.TestTemplateService;
14+
import org.elasticsearch.ElasticsearchParseException;
15+
import org.elasticsearch.ingest.TestTemplateService;
1316

1417
import java.util.Arrays;
1518
import java.util.HashMap;
1619
import java.util.List;
20+
import java.util.ArrayList;
1721
import java.util.Map;
1822

1923
import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_DEST_IP;
@@ -49,8 +53,11 @@ private Map<String, Object> buildEvent(String source, String destination) {
4953
}
5054

5155
public void testNoInternalNetworks() throws Exception {
52-
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testNetworkDirectionProcessor(buildEvent(), null));
53-
assertThat(e.getMessage(), containsString("unable to calculate network direction from document"));
56+
ElasticsearchParseException e = expectThrows(
57+
ElasticsearchParseException.class,
58+
() -> testNetworkDirectionProcessor(buildEvent(), null)
59+
);
60+
assertThat(e.getMessage(), containsString("either [internal_networks] or [internal_networks_field] must be specified"));
5461
}
5562

5663
public void testNoSource() throws Exception {
@@ -130,6 +137,27 @@ private void testNetworkDirectionProcessor(Map<String, Object> source, String[]
130137
testNetworkDirectionProcessor(source, internalNetworks, expectedDirection, false);
131138
}
132139

140+
public void testReadFromField() throws Exception {
141+
String processorTag = randomAlphaOfLength(10);
142+
Map<String, Object> source = buildEvent("192.168.1.1", "192.168.1.2");
143+
ArrayList<String> networks = new ArrayList<>();
144+
networks.add("public");
145+
source.put("some_field", networks);
146+
147+
Map<String, Object> config = new HashMap<>();
148+
config.put("internal_networks_field", "some_field");
149+
NetworkDirectionProcessor processor = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(
150+
null,
151+
processorTag,
152+
null,
153+
config
154+
);
155+
IngestDocument input = new IngestDocument(source, Map.of());
156+
IngestDocument output = processor.execute(input);
157+
String hash = output.getFieldValue(DEFAULT_TARGET, String.class);
158+
assertThat(hash, equalTo("external"));
159+
}
160+
133161
private void testNetworkDirectionProcessor(
134162
Map<String, Object> source,
135163
String[] internalNetworks,
@@ -140,14 +168,15 @@ private void testNetworkDirectionProcessor(
140168

141169
if (internalNetworks != null) networks = Arrays.asList(internalNetworks);
142170

143-
var processor = new NetworkDirectionProcessor(
171+
String processorTag = randomAlphaOfLength(10);
172+
Map<String, Object> config = new HashMap<>();
173+
config.put("internal_networks", networks);
174+
config.put("ignore_missing", ignoreMissing);
175+
NetworkDirectionProcessor processor = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(
144176
null,
177+
processorTag,
145178
null,
146-
DEFAULT_SOURCE_IP,
147-
DEFAULT_DEST_IP,
148-
DEFAULT_TARGET,
149-
networks,
150-
ignoreMissing
179+
config
151180
);
152181

153182
IngestDocument input = new IngestDocument(source, Map.of());

‎x-pack/plugin/ingest/src/main/java/org/elasticsearch/xpack/ingest/IngestPlugin.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.ingest;
9+
10+
import org.elasticsearch.ingest.Processor;
11+
import org.elasticsearch.plugins.Plugin;
12+
13+
import java.util.Map;
14+
15+
public class IngestPlugin extends Plugin implements org.elasticsearch.plugins.IngestPlugin {
16+
17+
@Override
18+
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
19+
return Map.of(
20+
UriPartsProcessor.TYPE,
21+
new UriPartsProcessor.Factory(),
22+
NetworkDirectionProcessor.TYPE,
23+
new NetworkDirectionProcessor.Factory(parameters.scriptService),
24+
CommunityIdProcessor.TYPE,
25+
new CommunityIdProcessor.Factory()
26+
);
27+
}
28+
}

0 commit comments

Comments
 (0)