Skip to content

Commit 060ad4d

Browse files
Andrew Stuckidanhermann
Andrew Stucki
authored and
danhermann
committed
Network direction processor supports dynamic internal networks specification (elastic#68712)
1 parent 6a7f6ae commit 060ad4d

File tree

5 files changed

+165
-34
lines changed

5 files changed

+165
-34
lines changed

‎docs/reference/ingest/processors/network-direction.asciidoc

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,18 @@ only the `internal_networks` option must be specified.
2121
| `source_ip` | no | `source.ip` | Field containing the source IP address.
2222
| `destination_ip` | no | `destination.ip` | Field containing the destination IP address.
2323
| `target_field` | no | `network.direction` | Output field for the network direction.
24-
| `internal_networks`| yes | | List of internal networks. Supports IPv4 and
25-
IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below.
24+
| `internal_networks`| yes * | | List of internal networks. Supports IPv4 and
25+
IPv6 addresses and ranges in CIDR notation. Also supports the named ranges listed below. These may be constructed with <<template-snippets,template snippets>>. * Must specify only one of `internal_networks` or `internal_networks_field`.
26+
| `internal_networks_field`| no | | A field on the given document to read the `internal_networks` configuration from.
2627
| `ignore_missing` | no | `true` | If `true` and any required fields are missing,
2728
the processor quietly exits without modifying the document.
2829

2930

3031
include::common-options.asciidoc[]
3132
|======
3233

34+
One of either `internal_networks` or `internal_networks_field` must be specified. If `internal_networks_field` is specified, it follows the behavior specified by `ignore_missing`.
35+
3336
[float]
3437
[[supported-named-network-ranges]]
3538
===== Supported named network ranges

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
8080
entry(HtmlStripProcessor.TYPE, new HtmlStripProcessor.Factory()),
8181
entry(CsvProcessor.TYPE, new CsvProcessor.Factory()),
8282
entry(UriPartsProcessor.TYPE, new UriPartsProcessor.Factory()),
83-
entry(NetworkDirectionProcessor.TYPE, new NetworkDirectionProcessor.Factory()),
83+
entry(NetworkDirectionProcessor.TYPE, new NetworkDirectionProcessor.Factory(parameters.scriptService)),
8484
entry(CommunityIdProcessor.TYPE, new CommunityIdProcessor.Factory()),
8585
entry(FingerprintProcessor.TYPE, new FingerprintProcessor.Factory())
8686
);

‎modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/NetworkDirectionProcessor.java

Lines changed: 67 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,17 @@
1414
import org.elasticsearch.ingest.ConfigurationUtils;
1515
import org.elasticsearch.ingest.IngestDocument;
1616
import org.elasticsearch.ingest.Processor;
17+
import org.elasticsearch.script.ScriptService;
18+
import org.elasticsearch.script.TemplateScript;
1719

1820
import java.net.InetAddress;
21+
import java.util.ArrayList;
1922
import java.util.Arrays;
2023
import java.util.List;
2124
import java.util.Map;
25+
import java.util.stream.Collectors;
2226

27+
import static org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException;
2328
import static org.elasticsearch.ingest.ConfigurationUtils.readBooleanProperty;
2429

2530
public class NetworkDirectionProcessor extends AbstractProcessor {
@@ -48,7 +53,8 @@ public class NetworkDirectionProcessor extends AbstractProcessor {
4853
private final String sourceIpField;
4954
private final String destinationIpField;
5055
private final String targetField;
51-
private final List<String> internalNetworks;
56+
private final List<TemplateScript.Factory> internalNetworks;
57+
private final String internalNetworksField;
5258
private final boolean ignoreMissing;
5359

5460
NetworkDirectionProcessor(
@@ -57,14 +63,16 @@ public class NetworkDirectionProcessor extends AbstractProcessor {
5763
String sourceIpField,
5864
String destinationIpField,
5965
String targetField,
60-
List<String> internalNetworks,
66+
List<TemplateScript.Factory> internalNetworks,
67+
String internalNetworksField,
6168
boolean ignoreMissing
6269
) {
6370
super(tag, description);
6471
this.sourceIpField = sourceIpField;
6572
this.destinationIpField = destinationIpField;
6673
this.targetField = targetField;
6774
this.internalNetworks = internalNetworks;
75+
this.internalNetworksField = internalNetworksField;
6876
this.ignoreMissing = ignoreMissing;
6977
}
7078

@@ -80,10 +88,14 @@ public String getTargetField() {
8088
return targetField;
8189
}
8290

83-
public List<String> getInternalNetworks() {
91+
public List<TemplateScript.Factory> getInternalNetworks() {
8492
return internalNetworks;
8593
}
8694

95+
public String getInternalNetworksField() {
96+
return internalNetworksField;
97+
}
98+
8799
public boolean getIgnoreMissing() {
88100
return ignoreMissing;
89101
}
@@ -103,9 +115,18 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
103115
return ingestDocument;
104116
}
105117

106-
private String getDirection(IngestDocument d) {
107-
if (internalNetworks == null) {
108-
return null;
118+
private String getDirection(IngestDocument d) throws Exception {
119+
List<String> networks = new ArrayList<>();
120+
121+
if (internalNetworksField != null) {
122+
@SuppressWarnings("unchecked")
123+
List<String> stringList = d.getFieldValue(internalNetworksField, networks.getClass(), ignoreMissing);
124+
if (stringList == null) {
125+
return null;
126+
}
127+
networks.addAll(stringList);
128+
} else {
129+
networks = internalNetworks.stream().map(network -> d.renderTemplate(network)).collect(Collectors.toList());
109130
}
110131

111132
String sourceIpAddrString = d.getFieldValue(sourceIpField, String.class, ignoreMissing);
@@ -118,8 +139,8 @@ private String getDirection(IngestDocument d) {
118139
return null;
119140
}
120141

121-
boolean sourceInternal = isInternal(sourceIpAddrString);
122-
boolean destinationInternal = isInternal(destIpAddrString);
142+
boolean sourceInternal = isInternal(networks, sourceIpAddrString);
143+
boolean destinationInternal = isInternal(networks, destIpAddrString);
123144

124145
if (sourceInternal && destinationInternal) {
125146
return DIRECTION_INTERNAL;
@@ -133,8 +154,8 @@ private String getDirection(IngestDocument d) {
133154
return DIRECTION_EXTERNAL;
134155
}
135156

136-
private boolean isInternal(String ip) {
137-
for (String network : internalNetworks) {
157+
private boolean isInternal(List<String> networks, String ip) {
158+
for (String network : networks) {
138159
if (inNetwork(ip, network)) {
139160
return true;
140161
}
@@ -227,31 +248,60 @@ public String getType() {
227248
}
228249

229250
public static final class Factory implements Processor.Factory {
230-
251+
private final ScriptService scriptService;
231252
static final String DEFAULT_SOURCE_IP = "source.ip";
232253
static final String DEFAULT_DEST_IP = "destination.ip";
233254
static final String DEFAULT_TARGET = "network.direction";
234255

256+
public Factory(ScriptService scriptService) {
257+
this.scriptService = scriptService;
258+
}
259+
235260
@Override
236261
public NetworkDirectionProcessor create(
237262
Map<String, Processor.Factory> registry,
238263
String processorTag,
239264
String description,
240265
Map<String, Object> config
241266
) throws Exception {
242-
String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP);
243-
String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP);
244-
String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET);
245-
List<String> internalNetworks = ConfigurationUtils.readList(TYPE, processorTag, config, "internal_networks");
246-
boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", true);
267+
final String sourceIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "source_ip", DEFAULT_SOURCE_IP);
268+
final String destIpField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "destination_ip", DEFAULT_DEST_IP);
269+
final String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field", DEFAULT_TARGET);
270+
final boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", true);
271+
272+
final List<String> internalNetworks = ConfigurationUtils.readOptionalList(TYPE, processorTag, config, "internal_networks");
273+
final String internalNetworksField = ConfigurationUtils.readOptionalStringProperty(
274+
TYPE,
275+
processorTag,
276+
config,
277+
"internal_networks_field"
278+
);
247279

280+
if (internalNetworks == null && internalNetworksField == null) {
281+
throw newConfigurationException(TYPE, processorTag, "internal_networks", "or [internal_networks_field] must be specified");
282+
}
283+
if (internalNetworks != null && internalNetworksField != null) {
284+
throw newConfigurationException(
285+
TYPE,
286+
processorTag,
287+
"internal_networks", "and [internal_networks_field] cannot both be used in the same processor"
288+
);
289+
}
290+
291+
List<TemplateScript.Factory> internalNetworkTemplates = null;
292+
if (internalNetworks != null) {
293+
internalNetworkTemplates = internalNetworks.stream()
294+
.map(n -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "internal_networks", n, scriptService))
295+
.collect(Collectors.toList());
296+
}
248297
return new NetworkDirectionProcessor(
249298
processorTag,
250299
description,
251300
sourceIpField,
252301
destIpField,
253302
targetField,
254-
internalNetworks,
303+
internalNetworkTemplates,
304+
internalNetworksField,
255305
ignoreMissing
256306
);
257307
}

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorFactoryTests.java

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,25 +10,28 @@
1010

1111
import org.elasticsearch.ElasticsearchParseException;
1212
import org.elasticsearch.test.ESTestCase;
13+
import org.elasticsearch.ingest.TestTemplateService;
1314
import org.junit.Before;
1415

1516
import java.util.ArrayList;
1617
import java.util.HashMap;
18+
import java.util.Collections;
1719
import java.util.List;
1820
import java.util.Map;
1921

2022
import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_DEST_IP;
2123
import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_SOURCE_IP;
2224
import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_TARGET;
2325
import static org.hamcrest.CoreMatchers.equalTo;
26+
import static org.hamcrest.Matchers.greaterThan;
2427

2528
public class NetworkDirectionProcessorFactoryTests extends ESTestCase {
2629

2730
private NetworkDirectionProcessor.Factory factory;
2831

2932
@Before
3033
public void init() {
31-
factory = new NetworkDirectionProcessor.Factory();
34+
factory = new NetworkDirectionProcessor.Factory(TestTemplateService.instance());
3235
}
3336

3437
public void testCreate() throws Exception {
@@ -52,7 +55,32 @@ public void testCreate() throws Exception {
5255
assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField));
5356
assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField));
5457
assertThat(networkProcessor.getTargetField(), equalTo(targetField));
55-
assertThat(networkProcessor.getInternalNetworks(), equalTo(internalNetworks));
58+
assertThat(networkProcessor.getInternalNetworks().size(), greaterThan(0));
59+
assertThat(networkProcessor.getInternalNetworks().get(0).newInstance(Collections.emptyMap()).execute(), equalTo("10.0.0.0/8"));
60+
assertThat(networkProcessor.getIgnoreMissing(), equalTo(ignoreMissing));
61+
}
62+
63+
public void testCreateInternalNetworksField() throws Exception {
64+
Map<String, Object> config = new HashMap<>();
65+
66+
String sourceIpField = randomAlphaOfLength(6);
67+
config.put("source_ip", sourceIpField);
68+
String destIpField = randomAlphaOfLength(6);
69+
config.put("destination_ip", destIpField);
70+
String targetField = randomAlphaOfLength(6);
71+
config.put("target_field", targetField);
72+
String internalNetworksField = randomAlphaOfLength(6);
73+
config.put("internal_networks_field", internalNetworksField);
74+
boolean ignoreMissing = randomBoolean();
75+
config.put("ignore_missing", ignoreMissing);
76+
77+
String processorTag = randomAlphaOfLength(10);
78+
NetworkDirectionProcessor networkProcessor = factory.create(null, processorTag, null, config);
79+
assertThat(networkProcessor.getTag(), equalTo(processorTag));
80+
assertThat(networkProcessor.getSourceIpField(), equalTo(sourceIpField));
81+
assertThat(networkProcessor.getDestinationIpField(), equalTo(destIpField));
82+
assertThat(networkProcessor.getTargetField(), equalTo(targetField));
83+
assertThat(networkProcessor.getInternalNetworksField(), equalTo(internalNetworksField));
5684
assertThat(networkProcessor.getIgnoreMissing(), equalTo(ignoreMissing));
5785
}
5886

@@ -63,7 +91,7 @@ public void testRequiredFields() throws Exception {
6391
factory.create(null, processorTag, null, config);
6492
fail("factory create should have failed");
6593
} catch (ElasticsearchParseException e) {
66-
assertThat(e.getMessage(), equalTo("[internal_networks] required property is missing"));
94+
assertThat(e.getMessage(), equalTo("[internal_networks] or [internal_networks_field] must be specified"));
6795
}
6896
}
6997

‎modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/NetworkDirectionProcessorTests.java

Lines changed: 61 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,16 @@
1010

1111
import org.elasticsearch.ingest.IngestDocument;
1212
import org.elasticsearch.test.ESTestCase;
13+
import org.elasticsearch.ingest.TestTemplateService;
14+
import org.elasticsearch.ElasticsearchParseException;
15+
import org.elasticsearch.ingest.TestTemplateService;
1316

1417
import java.util.Arrays;
1518
import java.util.HashMap;
1619
import java.util.List;
20+
import java.util.ArrayList;
1721
import java.util.Map;
1822

19-
import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_DEST_IP;
20-
import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_SOURCE_IP;
2123
import static org.elasticsearch.ingest.common.NetworkDirectionProcessor.Factory.DEFAULT_TARGET;
2224
import static org.hamcrest.Matchers.containsString;
2325
import static org.hamcrest.Matchers.equalTo;
@@ -49,8 +51,11 @@ private Map<String, Object> buildEvent(String source, String destination) {
4951
}
5052

5153
public void testNoInternalNetworks() throws Exception {
52-
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> testNetworkDirectionProcessor(buildEvent(), null));
53-
assertThat(e.getMessage(), containsString("unable to calculate network direction from document"));
54+
ElasticsearchParseException e = expectThrows(
55+
ElasticsearchParseException.class,
56+
() -> testNetworkDirectionProcessor(buildEvent(), null)
57+
);
58+
assertThat(e.getMessage(), containsString("[internal_networks] or [internal_networks_field] must be specified"));
5459
}
5560

5661
public void testNoSource() throws Exception {
@@ -130,6 +135,50 @@ private void testNetworkDirectionProcessor(Map<String, Object> source, String[]
130135
testNetworkDirectionProcessor(source, internalNetworks, expectedDirection, false);
131136
}
132137

138+
public void testReadFromField() throws Exception {
139+
String processorTag = randomAlphaOfLength(10);
140+
Map<String, Object> source = buildEvent("192.168.1.1", "192.168.1.2");
141+
ArrayList<String> networks = new ArrayList<>();
142+
networks.add("public");
143+
source.put("some_field", networks);
144+
145+
Map<String, Object> config = new HashMap<>();
146+
config.put("internal_networks_field", "some_field");
147+
NetworkDirectionProcessor processor = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(
148+
null,
149+
processorTag,
150+
null,
151+
config
152+
);
153+
IngestDocument input = new IngestDocument(source, Map.of());
154+
IngestDocument output = processor.execute(input);
155+
String hash = output.getFieldValue(DEFAULT_TARGET, String.class);
156+
assertThat(hash, equalTo("external"));
157+
}
158+
159+
public void testInternalNetworksAndField() throws Exception {
160+
String processorTag = randomAlphaOfLength(10);
161+
Map<String, Object> source = buildEvent("192.168.1.1", "192.168.1.2");
162+
ArrayList<String> networks = new ArrayList<>();
163+
networks.add("public");
164+
source.put("some_field", networks);
165+
Map<String, Object> config = new HashMap<>();
166+
config.put("internal_networks_field", "some_field");
167+
config.put("internal_networks", networks);
168+
ElasticsearchParseException e = expectThrows(
169+
ElasticsearchParseException.class,
170+
() -> new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(
171+
null,
172+
processorTag,
173+
null,
174+
config
175+
)
176+
);
177+
assertThat(e.getMessage(), containsString(
178+
"[internal_networks] and [internal_networks_field] cannot both be used in the same processor"
179+
));
180+
}
181+
133182
private void testNetworkDirectionProcessor(
134183
Map<String, Object> source,
135184
String[] internalNetworks,
@@ -140,17 +189,18 @@ private void testNetworkDirectionProcessor(
140189

141190
if (internalNetworks != null) networks = Arrays.asList(internalNetworks);
142191

143-
NetworkDirectionProcessor processor = new NetworkDirectionProcessor(
192+
String processorTag = randomAlphaOfLength(10);
193+
Map<String, Object> config = new HashMap<>();
194+
config.put("internal_networks", networks);
195+
config.put("ignore_missing", ignoreMissing);
196+
NetworkDirectionProcessor processor = new NetworkDirectionProcessor.Factory(TestTemplateService.instance()).create(
144197
null,
198+
processorTag,
145199
null,
146-
DEFAULT_SOURCE_IP,
147-
DEFAULT_DEST_IP,
148-
DEFAULT_TARGET,
149-
networks,
150-
ignoreMissing
200+
config
151201
);
152202

153-
IngestDocument input = new IngestDocument(source, org.elasticsearch.common.collect.Map.of());
203+
IngestDocument input = new IngestDocument(source, Map.of());
154204
IngestDocument output = processor.execute(input);
155205

156206
String hash = output.getFieldValue(DEFAULT_TARGET, String.class, ignoreMissing);

0 commit comments

Comments
 (0)