Skip to content

Commit 84771ce

Browse files
stevejgordonstevejgordon
authored
Add network community ID ingest processor (#5347)
1 parent e635a2b commit 84771ce

File tree

4 files changed

+237
-4
lines changed

4 files changed

+237
-4
lines changed

‎src/Nest/Ingest/ProcessorFormatter.cs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ internal class ProcessorFormatter : IJsonFormatter<IProcessor>
4747
{ "enrich", 31 },
4848
{ "csv", 32 },
4949
{ "uri_parts", 33 },
50-
{ "fingerprint", 34 }
50+
{ "fingerprint", 34 },
51+
{ "community_id", 35 }
5152
};
5253

5354
public IProcessor Deserialize(ref JsonReader reader, IJsonFormatterResolver formatterResolver)
@@ -173,6 +174,9 @@ public IProcessor Deserialize(ref JsonReader reader, IJsonFormatterResolver form
173174
case 34:
174175
processor = Deserialize<FingerprintProcessor>(ref reader, formatterResolver);
175176
break;
177+
case 35:
178+
processor = Deserialize<NetworkCommunityIdProcessor>(ref reader, formatterResolver);
179+
break;
176180
}
177181
}
178182
else
@@ -299,6 +303,9 @@ public void Serialize(ref JsonWriter writer, IProcessor value, IJsonFormatterRes
299303
case "fingerprint":
300304
Serialize<IFingerprintProcessor>(ref writer, value, formatterResolver);
301305
break;
306+
case "community_id":
307+
Serialize<INetworkCommunityIdProcessor>(ref writer, value, formatterResolver);
308+
break;
302309
default:
303310
var formatter = DynamicObjectResolver.ExcludeNullCamelCase.GetFormatter<IProcessor>();
304311
formatter.Serialize(ref writer, value, formatterResolver);

‎src/Nest/Ingest/Processors/NetworkCommunityIdProcessor.cs

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
using System;
2+
using System.Linq.Expressions;
3+
using System.Runtime.Serialization;
4+
using Elasticsearch.Net.Utf8Json;
5+
6+
namespace Nest
7+
{
8+
[InterfaceDataContract]
9+
public interface INetworkCommunityIdProcessor : IProcessor
10+
{
11+
[DataMember(Name = "destination_ip")]
12+
Field DestinationIp { get; set; }
13+
14+
[DataMember(Name = "destination_port")]
15+
Field DestinationPort { get; set; }
16+
17+
[DataMember(Name = "iana_number")]
18+
Field IanaNumber { get; set; }
19+
20+
[DataMember(Name = "icmp_type")]
21+
Field IcmpType { get; set; }
22+
23+
[DataMember(Name = "icmp_code")]
24+
Field IcmpCode { get; set; }
25+
26+
[DataMember(Name = "ignore_missing")]
27+
bool? IgnoreMissing { get; set; }
28+
29+
[DataMember(Name = "seed")]
30+
int? Seed { get; set; }
31+
32+
[DataMember(Name = "source_ip")]
33+
Field SourceIp { get; set; }
34+
35+
[DataMember(Name = "source_port")]
36+
Field SourcePort { get; set; }
37+
38+
[DataMember(Name = "target_field")]
39+
Field TargetField { get; set; }
40+
41+
[DataMember(Name = "transport")]
42+
Field Transport { get; set; }
43+
}
44+
45+
public class NetworkCommunityIdProcessor : ProcessorBase, INetworkCommunityIdProcessor
46+
{
47+
protected override string Name => "community_id";
48+
49+
/// <inheritdoc />
50+
public Field DestinationIp { get; set; }
51+
/// <inheritdoc />
52+
public Field DestinationPort { get; set; }
53+
/// <inheritdoc />
54+
public Field IanaNumber { get; set; }
55+
/// <inheritdoc />
56+
public Field IcmpType { get; set; }
57+
/// <inheritdoc />
58+
public Field IcmpCode { get; set; }
59+
/// <inheritdoc />
60+
public bool? IgnoreMissing { get; set; }
61+
/// <inheritdoc />
62+
public int? Seed { get; set; }
63+
/// <inheritdoc />
64+
public Field SourceIp { get; set; }
65+
/// <inheritdoc />
66+
public Field SourcePort { get; set; }
67+
/// <inheritdoc />
68+
public Field TargetField { get; set; }
69+
/// <inheritdoc />
70+
public Field Transport { get; set; }
71+
}
72+
73+
/// <inheritdoc cref="IFingerprintProcessor" />
74+
public class NetworkCommunityIdProcessorDescriptor<T>
75+
: ProcessorDescriptorBase<NetworkCommunityIdProcessorDescriptor<T>, INetworkCommunityIdProcessor>, INetworkCommunityIdProcessor
76+
where T : class
77+
{
78+
protected override string Name => "community_id";
79+
80+
Field INetworkCommunityIdProcessor.DestinationIp { get; set; }
81+
82+
Field INetworkCommunityIdProcessor.DestinationPort { get; set; }
83+
84+
Field INetworkCommunityIdProcessor.IanaNumber { get; set; }
85+
86+
Field INetworkCommunityIdProcessor.IcmpType { get; set; }
87+
88+
Field INetworkCommunityIdProcessor.IcmpCode { get; set; }
89+
90+
bool? INetworkCommunityIdProcessor.IgnoreMissing { get; set; }
91+
92+
int? INetworkCommunityIdProcessor.Seed { get; set; }
93+
94+
Field INetworkCommunityIdProcessor.SourceIp { get; set; }
95+
96+
Field INetworkCommunityIdProcessor.SourcePort { get; set; }
97+
98+
Field INetworkCommunityIdProcessor.TargetField { get; set; }
99+
100+
Field INetworkCommunityIdProcessor.Transport { get; set; }
101+
102+
/// <inheritdoc cref="INetworkCommunityIdProcessor.DestinationIp" />
103+
public NetworkCommunityIdProcessorDescriptor<T> DestinationIp(Field field) => Assign(field, (a, v) => a.DestinationIp = v);
104+
105+
/// <inheritdoc cref="INetworkCommunityIdProcessor.DestinationIp" />
106+
public NetworkCommunityIdProcessorDescriptor<T> DestinationIp<TValue>(Expression<Func<T, TValue>> objectPath) =>
107+
Assign(objectPath, (a, v) => a.DestinationIp = v);
108+
109+
/// <inheritdoc cref="INetworkCommunityIdProcessor.DestinationPort" />
110+
public NetworkCommunityIdProcessorDescriptor<T> DestinationPort(Field field) => Assign(field, (a, v) => a.DestinationPort = v);
111+
112+
/// <inheritdoc cref="INetworkCommunityIdProcessor.DestinationPort" />
113+
public NetworkCommunityIdProcessorDescriptor<T> DestinationPort<TValue>(Expression<Func<T, TValue>> objectPath) =>
114+
Assign(objectPath, (a, v) => a.DestinationPort = v);
115+
116+
/// <inheritdoc cref="INetworkCommunityIdProcessor.IanaNumber" />
117+
public NetworkCommunityIdProcessorDescriptor<T> IanaNumber(Field field) => Assign(field, (a, v) => a.IanaNumber = v);
118+
119+
/// <inheritdoc cref="INetworkCommunityIdProcessor.IanaNumber" />
120+
public NetworkCommunityIdProcessorDescriptor<T> IanaNumber<TValue>(Expression<Func<T, TValue>> objectPath) =>
121+
Assign(objectPath, (a, v) => a.IanaNumber = v);
122+
123+
/// <inheritdoc cref="INetworkCommunityIdProcessor.IcmpType" />
124+
public NetworkCommunityIdProcessorDescriptor<T> IcmpType(Field field) => Assign(field, (a, v) => a.IcmpType = v);
125+
126+
/// <inheritdoc cref="INetworkCommunityIdProcessor.IcmpType" />
127+
public NetworkCommunityIdProcessorDescriptor<T> IcmpType<TValue>(Expression<Func<T, TValue>> objectPath) =>
128+
Assign(objectPath, (a, v) => a.IcmpType = v);
129+
130+
/// <inheritdoc cref="INetworkCommunityIdProcessor.IcmpCode" />
131+
public NetworkCommunityIdProcessorDescriptor<T> IcmpCode(Field field) => Assign(field, (a, v) => a.IcmpCode = v);
132+
133+
/// <inheritdoc cref="INetworkCommunityIdProcessor.IcmpType" />
134+
public NetworkCommunityIdProcessorDescriptor<T> IcmpCode<TValue>(Expression<Func<T, TValue>> objectPath) =>
135+
Assign(objectPath, (a, v) => a.IcmpCode = v);
136+
137+
/// <inheritdoc cref="INetworkCommunityIdProcessor.IgnoreMissing" />
138+
public NetworkCommunityIdProcessorDescriptor<T> IgnoreMissing(bool? ignoreMissing = true) => Assign(ignoreMissing, (a, v) => a.IgnoreMissing = v);
139+
140+
/// <inheritdoc cref="INetworkCommunityIdProcessor.Seed" />
141+
public NetworkCommunityIdProcessorDescriptor<T> Seed(int? seed = null) => Assign(seed, (a, v) => a.Seed = v);
142+
143+
/// <inheritdoc cref="INetworkCommunityIdProcessor.SourceIp" />
144+
public NetworkCommunityIdProcessorDescriptor<T> SourceIp(Field field) => Assign(field, (a, v) => a.SourceIp = v);
145+
146+
/// <inheritdoc cref="INetworkCommunityIdProcessor.SourceIp" />
147+
public NetworkCommunityIdProcessorDescriptor<T> SourceIp<TValue>(Expression<Func<T, TValue>> objectPath) =>
148+
Assign(objectPath, (a, v) => a.SourceIp = v);
149+
150+
/// <inheritdoc cref="INetworkCommunityIdProcessor.SourcePort" />
151+
public NetworkCommunityIdProcessorDescriptor<T> SourcePort(Field field) => Assign(field, (a, v) => a.SourcePort = v);
152+
153+
/// <inheritdoc cref="INetworkCommunityIdProcessor.SourcePort" />
154+
public NetworkCommunityIdProcessorDescriptor<T> SourcePort<TValue>(Expression<Func<T, TValue>> objectPath) =>
155+
Assign(objectPath, (a, v) => a.SourcePort = v);
156+
157+
/// <inheritdoc cref="INetworkCommunityIdProcessor.TargetField" />
158+
public NetworkCommunityIdProcessorDescriptor<T> TargetField(Field field) => Assign(field, (a, v) => a.TargetField = v);
159+
160+
/// <inheritdoc cref="INetworkCommunityIdProcessor.TargetField" />
161+
public NetworkCommunityIdProcessorDescriptor<T> TargetField<TValue>(Expression<Func<T, TValue>> objectPath) =>
162+
Assign(objectPath, (a, v) => a.TargetField = v);
163+
164+
/// <inheritdoc cref="INetworkCommunityIdProcessor.Transport" />
165+
public NetworkCommunityIdProcessorDescriptor<T> Transport(Field field) => Assign(field, (a, v) => a.Transport = v);
166+
167+
/// <inheritdoc cref="INetworkCommunityIdProcessor.Transport" />
168+
public NetworkCommunityIdProcessorDescriptor<T> Transport<TValue>(Expression<Func<T, TValue>> objectPath) =>
169+
Assign(objectPath, (a, v) => a.Transport = v);
170+
}
171+
}

‎src/Nest/Ingest/ProcessorsDescriptor.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,5 +185,9 @@ public ProcessorsDescriptor UriParts<T>(Func<UriPartsProcessorDescriptor<T>, IUr
185185
/// <inheritdoc cref="IFingerprintProcessor"/>
186186
public ProcessorsDescriptor Fingerprint<T>(Func<FingerprintProcessorDescriptor<T>, IFingerprintProcessor> selector) where T : class =>
187187
Assign(selector, (a, v) => a.AddIfNotNull(v?.Invoke(new FingerprintProcessorDescriptor<T>())));
188+
189+
/// <inheritdoc cref="INetworkCommunityIdProcessor"/>
190+
public ProcessorsDescriptor NetworkCommunityId<T>(Func<NetworkCommunityIdProcessorDescriptor<T>, INetworkCommunityIdProcessor> selector) where T : class =>
191+
Assign(selector, (a, v) => a.AddIfNotNull(v?.Invoke(new NetworkCommunityIdProcessorDescriptor<T>())));
188192
}
189193
}

‎tests/Tests/Ingest/ProcessorAssertions.cs

Lines changed: 54 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
using Tests.Core.Extensions;
1313
using Tests.Core.Xunit;
1414
using Tests.Domain;
15+
using static Nest.Infer;
1516

1617
namespace Tests.Ingest
1718
{
@@ -189,7 +190,7 @@ public class Enrich : ProcessorAssertion
189190
public override IProcessor Initializer => new EnrichProcessor
190191
{
191192
PolicyName = PolicyName,
192-
Field = Infer.Field<Project>(f => f.Name),
193+
Field = Field<Project>(f => f.Name),
193194
TargetField = "target_field"
194195
};
195196

@@ -226,7 +227,7 @@ public class Foreach : ProcessorAssertion
226227

227228
public override IProcessor Initializer => new ForeachProcessor
228229
{
229-
Field = Infer.Field<Project>(p => p.Tags),
230+
Field = Field<Project>(p => p.Tags),
230231
Processor = new UppercaseProcessor
231232
{
232233
Field = "_value.name"
@@ -360,7 +361,7 @@ public class Set : ProcessorAssertion
360361
public override Func<ProcessorsDescriptor, IPromise<IList<IProcessor>>> Fluent =>
361362
d => d.Set<Project>(s => s.Field(p => p.Name).Value("foo"));
362363

363-
public override IProcessor Initializer => new SetProcessor { Field = Infer.Field<Project>(p => p.Name), Value = "foo" };
364+
public override IProcessor Initializer => new SetProcessor { Field = Field<Project>(p => p.Name), Value = "foo" };
364365

365366
public override object Json => new { field = "name", value = "foo" };
366367
public override string Key => "set";
@@ -732,5 +733,55 @@ public class Fingerprint : ProcessorAssertion
732733
public override object Json => new { fields = new[] { "labels" }, method = "MD5", salt = "ThisIsASalt!", target_field = "description", ignore_missing = true };
733734
public override string Key => "fingerprint";
734735
}
736+
737+
[SkipVersion("<7.12.0", "Uses the network community ID processor which was introduced in 7.12.0")]
738+
public class NetworkCommunityId : ProcessorAssertion
739+
{
740+
public override Func<ProcessorsDescriptor, IPromise<IList<IProcessor>>> Fluent => d => d
741+
.NetworkCommunityId<Project>(ud => ud
742+
.DestinationIp(f => f.LeadDeveloper.IpAddress)
743+
.DestinationPort("leadDeveloper.portNumber")
744+
.IanaNumber(f => f.Name)
745+
.IcmpType(f => f.Name)
746+
.IcmpCode(f => f.Name)
747+
.IgnoreMissing()
748+
.Seed(100)
749+
.SourceIp(f => f.LeadDeveloper.IpAddress)
750+
.SourcePort("leadDeveloper.portNumber")
751+
.TargetField(f => f.Description)
752+
.Transport(f => f.Name));
753+
754+
public override IProcessor Initializer => new NetworkCommunityIdProcessor
755+
{
756+
DestinationIp = Field<Project>(f => f.LeadDeveloper.IpAddress),
757+
DestinationPort = "leadDeveloper.portNumber",
758+
IanaNumber = "name",
759+
IcmpType = "name",
760+
IcmpCode = "name",
761+
IgnoreMissing = true,
762+
Seed = 100,
763+
SourceIp = Field<Project>(f => f.LeadDeveloper.IpAddress),
764+
SourcePort = "leadDeveloper.portNumber",
765+
TargetField = "description",
766+
Transport = "name"
767+
};
768+
769+
public override object Json => new
770+
{
771+
destination_ip = "leadDeveloper.ipAddress",
772+
destination_port = "leadDeveloper.portNumber",
773+
iana_number = "name",
774+
icmp_code = "name",
775+
icmp_type = "name",
776+
ignore_missing = true,
777+
seed = 100,
778+
source_ip = "leadDeveloper.ipAddress",
779+
source_port = "leadDeveloper.portNumber",
780+
target_field = "description",
781+
transport = "name"
782+
};
783+
784+
public override string Key => "community_id";
785+
}
735786
}
736787
}

0 commit comments

Comments
 (0)