Skip to content

Commit aae623f

Browse files
russcamrusscam
committed
Add inference processor
Relates: #4341, elastic/elasticsearch#49052 This commit adds the ingest inference processor.
1 parent 916214e commit aae623f

File tree

4 files changed

+282
-1
lines changed

4 files changed

+282
-1
lines changed

‎src/Nest/Ingest/ProcessorFormatter.cs

+7
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ internal class ProcessorFormatter : IJsonFormatter<IProcessor>
4242
{ "circle", 30 },
4343
{ "enrich", 31 },
4444
{ "csv", 32 },
45+
{ "inference", 33 },
4546
};
4647

4748
public IProcessor Deserialize(ref JsonReader reader, IJsonFormatterResolver formatterResolver)
@@ -161,6 +162,9 @@ public IProcessor Deserialize(ref JsonReader reader, IJsonFormatterResolver form
161162
case 32:
162163
processor = Deserialize<CsvProcessor>(ref reader, formatterResolver);
163164
break;
165+
case 33:
166+
processor = Deserialize<InferenceProcessor>(ref reader, formatterResolver);
167+
break;
164168
}
165169
}
166170
else
@@ -230,6 +234,9 @@ public void Serialize(ref JsonWriter writer, IProcessor value, IJsonFormatterRes
230234
case "gsub":
231235
Serialize<IGsubProcessor>(ref writer, value, formatterResolver);
232236
break;
237+
case "inference":
238+
Serialize<IInferenceProcessor>(ref writer, value, formatterResolver);
239+
break;
233240
case "join":
234241
Serialize<IJoinProcessor>(ref writer, value, formatterResolver);
235242
break;

‎src/Nest/Ingest/Processors/InferenceProcessor.cs

+219
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq.Expressions;
4+
using System.Runtime.Serialization;
5+
using Elasticsearch.Net;
6+
using Elasticsearch.Net.Utf8Json;
7+
using Nest;
8+
9+
namespace Nest
10+
{
11+
/// <summary>
12+
/// Uses a pre-trained data frame analytics model to infer against the data that is being ingested in the pipeline.
13+
/// <para />
14+
/// Available in Elasticsearch 7.6.0+ with at least basic license.
15+
/// </summary>
16+
[InterfaceDataContract]
17+
public interface IInferenceProcessor : IProcessor
18+
{
19+
/// <summary>
20+
/// The ID of the model to load and infer against.
21+
/// </summary>
22+
[DataMember(Name = "model_id")]
23+
string ModelId { get; set; }
24+
25+
/// <summary>
26+
/// Field added to incoming documents to contain results objects.
27+
/// </summary>
28+
[DataMember(Name ="target_field")]
29+
Field TargetField { get; set; }
30+
31+
/// <summary>
32+
/// Maps the document field names to the known field names of the model.
33+
/// </summary>
34+
[DataMember(Name = "field_mappings")]
35+
IDictionary<Field, Field> FieldMappings { get; set; }
36+
37+
/// <summary>
38+
/// Contains the inference type and its options.
39+
/// </summary>
40+
[DataMember(Name = "inference_config")]
41+
IInferenceConfig InferenceConfig { get; set; }
42+
}
43+
44+
/// <inheritdoc cref="IInferenceProcessor" />
45+
public class InferenceProcessor : ProcessorBase, IInferenceProcessor
46+
{
47+
/// <inheritdoc />
48+
public string ModelId { get; set; }
49+
50+
/// <inheritdoc />
51+
public Field TargetField { get; set; }
52+
53+
/// <inheritdoc />
54+
public IDictionary<Field, Field> FieldMappings { get; set; }
55+
56+
/// <inheritdoc />
57+
public IInferenceConfig InferenceConfig { get; set; }
58+
59+
protected override string Name => "inference";
60+
}
61+
62+
/// <inheritdoc cref="IInferenceProcessor" />
63+
public class InferenceProcessorDescriptor<T>
64+
: ProcessorDescriptorBase<InferenceProcessorDescriptor<T>, IInferenceProcessor>, IInferenceProcessor
65+
where T : class
66+
{
67+
protected override string Name => "inference";
68+
69+
Field IInferenceProcessor.TargetField { get; set; }
70+
string IInferenceProcessor.ModelId { get; set; }
71+
IInferenceConfig IInferenceProcessor.InferenceConfig { get; set; }
72+
IDictionary<Field, Field> IInferenceProcessor.FieldMappings { get; set; }
73+
74+
/// <inheritdoc cref="IInferenceProcessor.TargetField" />
75+
public InferenceProcessorDescriptor<T> TargetField(Field field) => Assign(field, (a, v) => a.TargetField = v);
76+
77+
/// <inheritdoc cref="IInferenceProcessor.TargetField" />
78+
public InferenceProcessorDescriptor<T> TargetField<TValue>(Expression<Func<T, TValue>> objectPath) =>
79+
Assign(objectPath, (a, v) => a.TargetField = v);
80+
81+
/// <inheritdoc cref="IInferenceProcessor.ModelId" />
82+
public InferenceProcessorDescriptor<T> ModelId(string modelId) =>
83+
Assign(modelId, (a, v) => a.ModelId = v);
84+
85+
/// <inheritdoc cref="IInferenceProcessor.ModelId" />
86+
public InferenceProcessorDescriptor<T> InferenceConfig(Func<InferenceConfigDescriptor<T>, IInferenceConfig> selector) =>
87+
Assign(selector, (a, v) => a.InferenceConfig = v.InvokeOrDefault(new InferenceConfigDescriptor<T>()));
88+
89+
/// <inheritdoc cref="IInferenceProcessor.FieldMappings" />
90+
public InferenceProcessorDescriptor<T> FieldMappings(Func<FluentDictionary<Field, Field>, FluentDictionary<Field, Field>> selector = null) =>
91+
Assign(selector, (a, v) => a.FieldMappings = v.InvokeOrDefault(new FluentDictionary<Field, Field>()));
92+
}
93+
94+
[ReadAs(typeof(InferenceConfig))]
95+
public interface IInferenceConfig
96+
{
97+
98+
[DataMember(Name = "regression")]
99+
IRegressionInferenceConfig Regression { get; set; }
100+
101+
[DataMember(Name = "classification")]
102+
IClassificationInferenceConfig Classification { get; set; }
103+
}
104+
105+
public class InferenceConfig
106+
: IInferenceConfig
107+
{
108+
public IRegressionInferenceConfig Regression { get; set; }
109+
110+
public IClassificationInferenceConfig Classification { get; set; }
111+
}
112+
113+
public class InferenceConfigDescriptor<T> : DescriptorBase<InferenceConfigDescriptor<T>, IInferenceConfig>, IInferenceConfig
114+
{
115+
IRegressionInferenceConfig IInferenceConfig.Regression { get; set; }
116+
IClassificationInferenceConfig IInferenceConfig.Classification { get; set; }
117+
118+
public InferenceConfigDescriptor<T> Regression(Func<RegressionInferenceConfigDescriptor<T>, IRegressionInferenceConfig> selector) =>
119+
Assign(selector, (a, v) => a.Regression = v.InvokeOrDefault(new RegressionInferenceConfigDescriptor<T>()));
120+
121+
public InferenceConfigDescriptor<T> Classification(Func<ClassificationInferenceConfigDescriptor<T>, IClassificationInferenceConfig> selector) =>
122+
Assign(selector, (a, v) => a.Classification = v.InvokeOrDefault(new ClassificationInferenceConfigDescriptor<T>()));
123+
}
124+
125+
[ReadAs(typeof(RegressionInferenceConfig))]
126+
public interface IRegressionInferenceConfig
127+
{
128+
/// <summary>
129+
/// Specifies the field to which the inference prediction is written. Defaults to <c>predicted_value</c>.
130+
/// </summary>
131+
[DataMember(Name = "results_field")]
132+
Field ResultsField { get; set; }
133+
}
134+
135+
public class RegressionInferenceConfig : IRegressionInferenceConfig
136+
{
137+
/// <summary>
138+
/// Specifies the field to which the inference prediction is written. Defaults to <c>predicted_value</c>.
139+
/// </summary>
140+
public Field ResultsField { get; set; }
141+
}
142+
143+
public class RegressionInferenceConfigDescriptor<T>
144+
: DescriptorBase<RegressionInferenceConfigDescriptor<T>, IRegressionInferenceConfig>, IRegressionInferenceConfig
145+
{
146+
Field IRegressionInferenceConfig.ResultsField { get; set; }
147+
148+
/// <inheritdoc cref="IRegressionInferenceConfig.ResultsField" />
149+
public RegressionInferenceConfigDescriptor<T> ResultsField(Field field) => Assign(field, (a, v) => a.ResultsField = v);
150+
151+
/// <inheritdoc cref="IRegressionInferenceConfig.ResultsField" />
152+
public RegressionInferenceConfigDescriptor<T> ResultsField<TValue>(Expression<Func<T, TValue>> objectPath) =>
153+
Assign(objectPath, (a, v) => a.ResultsField = v);
154+
}
155+
156+
[ReadAs(typeof(ClassificationInferenceConfig))]
157+
public interface IClassificationInferenceConfig
158+
{
159+
/// <summary>
160+
/// Specifies the field to which the inference prediction is written. Defaults to <c>predicted_value</c>.
161+
/// </summary>
162+
[DataMember(Name = "results_field")]
163+
Field ResultsField { get; set; }
164+
165+
/// <summary>
166+
/// Specifies the number of top class predictions to return. Defaults to <c>0</c>.
167+
/// </summary>
168+
[DataMember(Name = "num_top_classes")]
169+
int? NumTopClasses { get; set; }
170+
171+
/// <summary>
172+
/// Specifies the field to which the top classes are written. Defaults to <c>top_classes</c>.
173+
/// </summary>
174+
[DataMember(Name = "top_classes_results_field")]
175+
Field TopClassesResultsField { get; set; }
176+
}
177+
178+
public class ClassificationInferenceConfig : IClassificationInferenceConfig
179+
{
180+
/// <summary>
181+
/// Specifies the field to which the inference prediction is written. Defaults to <c>predicted_value</c>.
182+
/// </summary>
183+
public Field ResultsField { get; set; }
184+
185+
/// <summary>
186+
/// Specifies the number of top class predictions to return. Defaults to <c>0</c>.
187+
/// </summary>
188+
public int? NumTopClasses { get; set; }
189+
190+
/// <summary>
191+
/// Specifies the field to which the top classes are written. Defaults to <c>top_classes</c>.
192+
/// </summary>
193+
public Field TopClassesResultsField { get; set; }
194+
}
195+
196+
public class ClassificationInferenceConfigDescriptor<T> : DescriptorBase<ClassificationInferenceConfigDescriptor<T>, IClassificationInferenceConfig>, IClassificationInferenceConfig
197+
{
198+
Field IClassificationInferenceConfig.ResultsField { get; set; }
199+
int? IClassificationInferenceConfig.NumTopClasses { get; set; }
200+
Field IClassificationInferenceConfig.TopClassesResultsField { get; set; }
201+
202+
/// <inheritdoc cref="IClassificationInferenceConfig.ResultsField" />
203+
public ClassificationInferenceConfigDescriptor<T> ResultsField(Field field) => Assign(field, (a, v) => a.ResultsField = v);
204+
205+
/// <inheritdoc cref="IClassificationInferenceConfig.ResultsField" />
206+
public ClassificationInferenceConfigDescriptor<T> ResultsField<TValue>(Expression<Func<T, TValue>> objectPath) =>
207+
Assign(objectPath, (a, v) => a.ResultsField = v);
208+
209+
/// <inheritdoc cref="IClassificationInferenceConfig.NumTopClasses" />
210+
public ClassificationInferenceConfigDescriptor<T> NumTopClasses(int? numTopClasses) => Assign(numTopClasses, (a, v) => a.NumTopClasses = v);
211+
212+
/// <inheritdoc cref="IClassificationInferenceConfig.TopClassesResultsField" />
213+
public ClassificationInferenceConfigDescriptor<T> TopClassesResultsField(Field field) => Assign(field, (a, v) => a.TopClassesResultsField = v);
214+
215+
/// <inheritdoc cref="IClassificationInferenceConfig.TopClassesResultsField" />
216+
public ClassificationInferenceConfigDescriptor<T> TopClassesResultsField<TValue>(Expression<Func<T, TValue>> objectPath) =>
217+
Assign(objectPath, (a, v) => a.TopClassesResultsField = v);
218+
}
219+
}

‎src/Nest/Ingest/ProcessorsDescriptor.cs

+4
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,10 @@ public ProcessorsDescriptor Grok<T>(Func<GrokProcessorDescriptor<T>, IGrokProces
7979
public ProcessorsDescriptor Gsub<T>(Func<GsubProcessorDescriptor<T>, IGsubProcessor> selector) where T : class =>
8080
Assign(selector, (a, v) => a.AddIfNotNull(v?.Invoke(new GsubProcessorDescriptor<T>())));
8181

82+
/// <inheritdoc cref="IInferenceProcessor"/>
83+
public ProcessorsDescriptor Inference<T>(Func<InferenceProcessorDescriptor<T>, IInferenceProcessor> selector) where T : class =>
84+
Assign(selector, (a, v) => a.AddIfNotNull(v?.Invoke(new InferenceProcessorDescriptor<T>())));
85+
8286
/// <inheritdoc cref="IJoinProcessor"/>
8387
public ProcessorsDescriptor Join<T>(Func<JoinProcessorDescriptor<T>, IJoinProcessor> selector) where T : class =>
8488
Assign(selector, (a, v) => a.AddIfNotNull(v?.Invoke(new JoinProcessorDescriptor<T>())));

‎tests/Tests/Ingest/ProcessorAssertions.cs

+52-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ public abstract class ProcessorAssertion : IProcessorAssertion
2727
public abstract string Key { get; }
2828
}
2929

30-
3130
public static class ProcessorAssertions
3231
{
3332
public static IEnumerable<IProcessorAssertion> All =>
@@ -279,6 +278,58 @@ public class Gsub : ProcessorAssertion
279278
public override string Key => "gsub";
280279
}
281280

281+
[SkipVersion("<7.6.0", "Introduced in Elasticsearch 7.6.0+")]
282+
public class Inference : ProcessorAssertion
283+
{
284+
public override Func<ProcessorsDescriptor, IPromise<IList<IProcessor>>> Fluent => d => d
285+
.Inference<Project>(c => c
286+
.TargetField(p => p.Name)
287+
.ModelId("model_id")
288+
.FieldMappings()
289+
.InferenceConfig(i => i
290+
.Classification(cc => cc
291+
.ResultsField("results")
292+
.NumTopClasses(10)
293+
.TopClassesResultsField("topClasses")
294+
)
295+
)
296+
);
297+
298+
public override IProcessor Initializer => new InferenceProcessor
299+
{
300+
TargetField = "name",
301+
ModelId = "model_id",
302+
FieldMappings = new Dictionary<Field, Field>(),
303+
InferenceConfig = new InferenceConfig
304+
{
305+
Classification = new ClassificationInferenceConfig
306+
{
307+
ResultsField = "results",
308+
NumTopClasses = 10,
309+
TopClassesResultsField = "topClasses"
310+
}
311+
}
312+
};
313+
314+
public override object Json => new
315+
{
316+
target_field = "name",
317+
model_id = "model_id",
318+
field_mappings = new {},
319+
inference_config = new
320+
{
321+
classification = new
322+
{
323+
results_field = "results",
324+
num_top_classes = 10,
325+
top_classes_results_field = "topClasses"
326+
}
327+
}
328+
};
329+
330+
public override string Key => "inference";
331+
}
332+
282333
public class Join : ProcessorAssertion
283334
{
284335
public override Func<ProcessorsDescriptor, IPromise<IList<IProcessor>>> Fluent =>

0 commit comments

Comments
 (0)