Skip to content

Commit ad4f02c

Browse files
Mpdreamzrusscam
Mpdreamz
authored and
russcam
committed
Add KeyValueProcessor tests and add support for trimming properties (#3439)
 (cherry picked from commit b3e286f)
1 parent 258800d commit ad4f02c

File tree

3 files changed

+121
-26
lines changed

3 files changed

+121
-26
lines changed

‎src/Nest/Ingest/Processors/KeyValueProcessor.cs

Lines changed: 49 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
using System;
2-
using System.Collections;
32
using System.Collections.Generic;
4-
using System.Linq;
53
using System.Linq.Expressions;
64
using Newtonsoft.Json;
75

@@ -11,33 +9,23 @@ namespace Nest
119
[JsonConverter(typeof(ProcessorJsonConverter<KeyValueProcessor>))]
1210
public interface IKeyValueProcessor : IProcessor
1311
{
14-
/// <summary>
15-
/// The field to be parsed
16-
/// </summary>
12+
/// <summary> The field to be parsed </summary>
1713
[JsonProperty("field")]
1814
Field Field { get; set; }
1915

20-
/// <summary>
21-
/// The field to insert the extracted keys into. Defaults to the root of the document
22-
/// </summary>
16+
/// <summary> The field to insert the extracted keys into. Defaults to the root of the document </summary>
2317
[JsonProperty("target_field")]
2418
Field TargetField { get; set; }
2519

26-
/// <summary>
27-
/// Regex pattern to use for splitting key-value pairs
28-
/// </summary>
20+
/// <summary> Regex pattern to use for splitting key-value pairs </summary>
2921
[JsonProperty("field_split")]
3022
string FieldSplit { get; set; }
3123

32-
/// <summary>
33-
/// Regex pattern to use for splitting the key from the value within a key-value pair
34-
/// </summary>
24+
/// <summary> Regex pattern to use for splitting the key from the value within a key-value pair </summary>
3525
[JsonProperty("value_split")]
3626
string ValueSplit { get; set; }
3727

38-
/// <summary>
39-
/// List of keys to filter and insert into document. Defaults to including all keys
40-
/// </summary>
28+
/// <summary> List of keys to filter and insert into document. Defaults to including all keys </summary>
4129
[JsonProperty("include_keys")]
4230
IEnumerable<string> IncludeKeys { get; set; }
4331

@@ -46,8 +34,21 @@ public interface IKeyValueProcessor : IProcessor
4634
/// </summary>
4735
[JsonProperty("ignore_missing")]
4836
bool? IgnoreMissing { get; set; }
37+
38+
/// <summary> String of characters to trim from extracted keys </summary>
39+
[JsonProperty("trim_key")]
40+
string TrimKey { get; set; }
41+
42+
/// <summary> String of characters to trim from extracted values </summary>
43+
[JsonProperty("trim_value")]
44+
string TrimValue { get; set; }
45+
46+
/// <summary> If true strip brackets (), &lt;&gt;, [] as well as quotes ' and " from extracted values </summary>
47+
[JsonProperty("strip_brackets")]
48+
bool? StripBrackets { get; set; }
4949
}
5050

51+
/// <inheritdoc cref="IKeyValueProcessor"/>
5152
public class KeyValueProcessor : ProcessorBase, IKeyValueProcessor
5253
{
5354
protected override string Name => "kv";
@@ -69,8 +70,18 @@ public class KeyValueProcessor : ProcessorBase, IKeyValueProcessor
6970

7071
/// <inheritdoc/>
7172
public bool? IgnoreMissing { get; set; }
73+
74+
/// <inheritdoc/>
75+
public string TrimKey { get; set; }
76+
77+
/// <inheritdoc/>
78+
public string TrimValue { get; set; }
79+
80+
/// <inheritdoc/>
81+
public bool? StripBrackets { get; set; }
7282
}
7383

84+
/// <inheritdoc cref="IKeyValueProcessor"/>
7485
public class KeyValueProcessorDescriptor<T> : ProcessorDescriptorBase<KeyValueProcessorDescriptor<T>, IKeyValueProcessor>, IKeyValueProcessor
7586
where T : class
7687
{
@@ -82,33 +93,45 @@ public class KeyValueProcessorDescriptor<T> : ProcessorDescriptorBase<KeyValuePr
8293
string IKeyValueProcessor.ValueSplit { get; set; }
8394
IEnumerable<string> IKeyValueProcessor.IncludeKeys { get; set; }
8495
bool? IKeyValueProcessor.IgnoreMissing { get; set; }
96+
string IKeyValueProcessor.TrimKey { get; set; }
97+
string IKeyValueProcessor.TrimValue { get; set; }
98+
bool? IKeyValueProcessor.StripBrackets { get; set; }
8599

86-
/// <inheritdoc/>
100+
/// <inheritdoc cref="IKeyValueProcessor.Field"/>
87101
public KeyValueProcessorDescriptor<T> Field(Field field) => Assign(a => a.Field = field);
88102

89-
/// <inheritdoc/>
103+
/// <inheritdoc cref="IKeyValueProcessor.Field"/>
90104
public KeyValueProcessorDescriptor<T> Field(Expression<Func<T, object>> objectPath) => Assign(a => a.Field = objectPath);
91105

92-
/// <inheritdoc/>
106+
/// <inheritdoc cref="IKeyValueProcessor.TargetField"/>
93107
public KeyValueProcessorDescriptor<T> TargetField(Field field) => Assign(a => a.TargetField = field);
94108

95-
/// <inheritdoc/>
109+
/// <inheritdoc cref="IKeyValueProcessor.TargetField"/>
96110
public KeyValueProcessorDescriptor<T> TargetField(Expression<Func<T, object>> objectPath) => Assign(a => a.TargetField = objectPath);
97111

98-
/// <inheritdoc/>
112+
/// <inheritdoc cref="IKeyValueProcessor.FieldSplit"/>
99113
public KeyValueProcessorDescriptor<T> FieldSplit(string split) => Assign(a => a.FieldSplit = split);
100114

101-
/// <inheritdoc/>
115+
/// <inheritdoc cref="IKeyValueProcessor.ValueSplit"/>
102116
public KeyValueProcessorDescriptor<T> ValueSplit(string split) => Assign(a => a.ValueSplit = split);
103117

104-
/// <inheritdoc/>
118+
/// <inheritdoc cref="IKeyValueProcessor.IgnoreMissing"/>
105119
public KeyValueProcessorDescriptor<T> IgnoreMissing(bool? ignoreMissing = true) => Assign(a => a.IgnoreMissing = ignoreMissing);
106120

107-
/// <inheritdoc/>
121+
/// <inheritdoc cref="IKeyValueProcessor.IncludeKeys"/>
108122
public KeyValueProcessorDescriptor<T> IncludeKeys(IEnumerable<string> includeKeys) => Assign(a => a.IncludeKeys = includeKeys);
109123

110-
/// <inheritdoc/>
124+
/// <inheritdoc cref="IKeyValueProcessor.IncludeKeys"/>
111125
public KeyValueProcessorDescriptor<T> IncludeKeys(params string[] includeKeys) => Assign(a => a.IncludeKeys = includeKeys);
126+
127+
/// <inheritdoc cref="IKeyValueProcessor.TrimKey"/>
128+
public KeyValueProcessorDescriptor<T> TrimKey(string trimKeys) => Assign(a => a.TrimKey = trimKeys);
129+
130+
/// <inheritdoc cref="IKeyValueProcessor.TrimValue"/>
131+
public KeyValueProcessorDescriptor<T> TrimValue(string trimValues) => Assign(a => a.TrimValue = trimValues);
132+
133+
/// <inheritdoc cref="IKeyValueProcessor.StripBrackets"/>
134+
public KeyValueProcessorDescriptor<T> StripBrackets(bool? skip = true) => Assign(a => a.StripBrackets = skip);
112135
}
113136

114137
}

‎src/Nest/Ingest/ProcessorsDescriptor.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,14 @@ public ProcessorsDescriptor Json<T>(Func<JsonProcessorDescriptor<T>, IJsonProces
124124
public ProcessorsDescriptor UserAgent<T>(Func<UserAgentProcessorDescriptor<T>, IUserAgentProcessor> selector) where T : class =>
125125
Assign(a => a.AddIfNotNull(selector?.Invoke(new UserAgentProcessorDescriptor<T>())));
126126

127+
[Obsolete("This method takes the wrong descriptor please use Kv")]
127128
public ProcessorsDescriptor KeyValue<T>(Func<UserAgentProcessorDescriptor<T>, IUserAgentProcessor> selector) where T : class =>
128129
Assign(a => a.AddIfNotNull(selector?.Invoke(new UserAgentProcessorDescriptor<T>())));
129130

131+
/// <inheritdoc cref="IKeyValueProcessor"/>
132+
public ProcessorsDescriptor Kv<T>(Func<KeyValueProcessorDescriptor<T>, IKeyValueProcessor> selector) where T : class =>
133+
Assign(a => a.AddIfNotNull(selector?.Invoke(new KeyValueProcessorDescriptor<T>())));
134+
130135
/// <summary>
131136
/// URL-decodes a string
132137
/// </summary>

‎src/Tests/Tests/Ingest/ProcessorAssertions.cs

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,5 +341,72 @@ public class UrlDecode : ProcessorAssertion
341341
.IgnoreMissing()
342342
);
343343
}
344+
345+
public class KeyValue : ProcessorAssertion
346+
{
347+
public override string Key => "kv";
348+
349+
public override object Json => new
350+
{
351+
field = "description",
352+
ignore_missing = true,
353+
field_split = "_",
354+
value_split = " "
355+
};
356+
357+
public override IProcessor Initializer => new KeyValueProcessor
358+
{
359+
Field = "description",
360+
FieldSplit = "_",
361+
ValueSplit = " ",
362+
IgnoreMissing = true
363+
};
364+
365+
public override Func<ProcessorsDescriptor, IPromise<IList<IProcessor>>> Fluent => d => d
366+
.Kv<Project>(ud => ud
367+
.Field(p => p.Description)
368+
.FieldSplit("_")
369+
.ValueSplit(" ")
370+
.IgnoreMissing()
371+
);
372+
}
373+
[SkipVersion("<6.4.0", "trimming options were introduced later")]
374+
public class KeyValueTrimming : ProcessorAssertion
375+
{
376+
public override string Key => "kv";
377+
378+
public override object Json => new
379+
{
380+
field = "description",
381+
ignore_missing = true,
382+
field_split = "_",
383+
value_split = " ",
384+
trim_key = "xyz",
385+
trim_value = "abc",
386+
strip_brackets = true
387+
};
388+
389+
public override IProcessor Initializer => new KeyValueProcessor
390+
{
391+
Field = "description",
392+
FieldSplit = "_",
393+
ValueSplit = " ",
394+
TrimKey = "xyz",
395+
TrimValue = "abc",
396+
StripBrackets = true,
397+
IgnoreMissing = true
398+
};
399+
400+
public override Func<ProcessorsDescriptor, IPromise<IList<IProcessor>>> Fluent => d => d
401+
.Kv<Project>(ud => ud
402+
.Field(p => p.Description)
403+
.FieldSplit("_")
404+
.ValueSplit(" ")
405+
.TrimKey("xyz")
406+
.TrimValue("abc")
407+
.StripBrackets()
408+
.IgnoreMissing()
409+
);
410+
}
344411
}
345412
}

0 commit comments

Comments
 (0)