Skip to content

Commit 49e92e2

Browse files
russcamrusscam
committed
Add support for flattened datatype (#4058)
* Add support for flattened datatype Relates #4001 This commit adds support for the flattened data type introduced in 7.3.0 and basic. (cherry picked from commit fe9b212)
1 parent a5ff75f commit 49e92e2

File tree

16 files changed

+544
-82
lines changed

16 files changed

+544
-82
lines changed

‎src/Nest/Mapping/Types/Complex/Flattened/FlattenedAttribute.cs

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
using System.Xml.Schema;
2+
3+
namespace Nest
4+
{
5+
/// <inheritdoc cref="IFlattenedProperty" />
6+
public class FlattenedAttribute : ElasticsearchPropertyAttributeBase, IFlattenedProperty
7+
{
8+
public FlattenedAttribute() : base(FieldType.Flattened) { }
9+
10+
private IFlattenedProperty Self => this;
11+
12+
double? IFlattenedProperty.Boost { get; set; }
13+
int? IFlattenedProperty.DepthLimit { get; set; }
14+
bool? IFlattenedProperty.DocValues { get; set; }
15+
bool? IFlattenedProperty.EagerGlobalOrdinals { get; set; }
16+
int? IFlattenedProperty.IgnoreAbove { get; set; }
17+
bool? IFlattenedProperty.Index { get; set; }
18+
IndexOptions? IFlattenedProperty.IndexOptions { get; set; }
19+
bool? IFlattenedProperty.SplitQueriesOnWhitespace { get; set; }
20+
21+
/// <inheritdoc cref="IFlattenedProperty.Boost" />
22+
public double Boost
23+
{
24+
get => Self.Boost.GetValueOrDefault(1);
25+
set => Self.Boost = value;
26+
}
27+
28+
/// <inheritdoc cref="IFlattenedProperty.DepthLimit" />
29+
public int DepthLimit
30+
{
31+
get => Self.DepthLimit.GetValueOrDefault(20);
32+
set => Self.DepthLimit = value;
33+
}
34+
35+
/// <inheritdoc cref="IFlattenedProperty.DocValues" />
36+
public bool DocValues
37+
{
38+
get => Self.DocValues.GetValueOrDefault(true);
39+
set => Self.DocValues = value;
40+
}
41+
42+
/// <inheritdoc cref="IFlattenedProperty.EagerGlobalOrdinals" />
43+
public bool EagerGlobalOrdinals
44+
{
45+
get => Self.EagerGlobalOrdinals.GetValueOrDefault(false);
46+
set => Self.EagerGlobalOrdinals = value;
47+
}
48+
49+
/// <inheritdoc cref="IFlattenedProperty.IgnoreAbove" />
50+
public int IgnoreAbove
51+
{
52+
get => Self.IgnoreAbove.GetValueOrDefault(-1);
53+
set => Self.IgnoreAbove = value;
54+
}
55+
56+
/// <inheritdoc cref="IFlattenedProperty.Index" />
57+
public bool Index
58+
{
59+
get => Self.Index.GetValueOrDefault(true);
60+
set => Self.Index = value;
61+
}
62+
63+
/// <inheritdoc cref="IFlattenedProperty.IndexOptions" />
64+
public IndexOptions IndexOptions
65+
{
66+
get => Self.IndexOptions.GetValueOrDefault(IndexOptions.Docs);
67+
set => Self.IndexOptions = value;
68+
}
69+
70+
/// <inheritdoc cref="IFlattenedProperty.SplitQueriesOnWhitespace" />
71+
public bool SplitQueriesOnWhitespace
72+
{
73+
get => Self.SplitQueriesOnWhitespace.GetValueOrDefault(false);
74+
set => Self.SplitQueriesOnWhitespace = value;
75+
}
76+
77+
/// <inheritdoc />
78+
public string NullValue { get; set; }
79+
80+
/// <inheritdoc />
81+
public string Similarity { get; set; }
82+
}
83+
}

‎src/Nest/Mapping/Types/Complex/Flattened/FlattenedProperty.cs

+175
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
using System.Diagnostics;
2+
using System.Runtime.Serialization;
3+
using Elasticsearch.Net.Utf8Json;
4+
5+
namespace Nest
6+
{
7+
/// <summary>
8+
/// By default, each subfield in an object is mapped and indexed separately.
9+
/// If the names or types of the subfields are not known in advance, then they are mapped dynamically.
10+
/// The flattened type provides an alternative approach, where the entire object is mapped as a single field.
11+
/// Given an object, the flattened mapping will parse out its leaf values and index them into one
12+
/// field as keywords. The object's contents can then be searched through simple queries and aggregations.
13+
/// <para></para>
14+
/// Available in Elasticsearch 7.3.0+ with at least basic license level
15+
/// </summary>
16+
[InterfaceDataContract]
17+
public interface IFlattenedProperty : IProperty
18+
{
19+
/// <summary>
20+
/// Mapping field-level query time boosting. Accepts a floating point number, defaults to 1.0.
21+
/// </summary>
22+
[DataMember(Name = "boost")]
23+
double? Boost { get; set; }
24+
25+
/// <summary>
26+
/// The maximum allowed depth of the flattened object field,
27+
/// in terms of nested inner objects. If a flattened object field exceeds this limit,
28+
/// then an error will be thrown. Defaults to <c>20</c>.
29+
/// </summary>
30+
[DataMember(Name = "depth_limit")]
31+
int? DepthLimit { get; set; }
32+
33+
/// <summary>
34+
/// Whether to persist the value at index time in a columnar data structure (referred to as doc_values in Lucene)
35+
/// which makes the value available for efficient sorting and aggregations. Default is <c>true</c>.
36+
/// </summary>
37+
[DataMember(Name = "doc_values")]
38+
bool? DocValues { get; set; }
39+
40+
/// <summary>
41+
/// Should global ordinals be loaded eagerly on refresh? Accepts true or false (default).
42+
/// Enabling this is a good idea on fields that are frequently used for terms aggregations.
43+
/// </summary>
44+
[DataMember(Name = "eager_global_ordinals")]
45+
bool? EagerGlobalOrdinals { get; set; }
46+
47+
/// <summary>
48+
/// Leaf values longer than this limit will not be indexed. By default, there is no limit and all values will be indexed.
49+
/// Note that this limit applies to the leaf values within the flattened object field, and not the length of the entire
50+
/// field.
51+
/// </summary>
52+
[DataMember(Name = "ignore_above")]
53+
int? IgnoreAbove { get; set; }
54+
55+
/// <summary>
56+
/// Should the field be searchable? Accepts <c>true</c> (default) and <c>false</c>.
57+
/// </summary>
58+
[DataMember(Name = "index")]
59+
bool? Index { get; set; }
60+
61+
/// <summary>
62+
/// What information should be stored in the index for scoring purposes.
63+
/// Defaults to docs but can also be set to freqs to take term frequency into account when computing scores.
64+
/// </summary>
65+
[DataMember(Name = "index_options")]
66+
IndexOptions? IndexOptions { get; set; }
67+
68+
/// <summary>
69+
/// A string value which is substituted for any explicit null values within the flattened
70+
/// object field. Defaults to null, which means null fields are treated as if it were missing.
71+
/// </summary>
72+
[DataMember(Name = "null_value")]
73+
string NullValue { get; set; }
74+
75+
/// <summary>
76+
/// Which relevancy scoring algorithm or similarity should be used.
77+
/// Defaults to BM25
78+
/// </summary>
79+
[DataMember(Name = "similarity")]
80+
string Similarity { get; set; }
81+
82+
/// <summary> Whether full text queries should split the input on whitespace when building a query for this field. </summary>
83+
[DataMember(Name = "split_queries_on_whitespace")]
84+
bool? SplitQueriesOnWhitespace { get; set; }
85+
}
86+
87+
/// <inheritdoc cref="IFlattenedProperty" />
88+
[DebuggerDisplay("{DebugDisplay}")]
89+
public class FlattenedProperty : PropertyBase, IFlattenedProperty
90+
{
91+
public FlattenedProperty() : base(FieldType.Flattened) { }
92+
93+
/// <inheritdoc />
94+
public double? Boost { get; set; }
95+
96+
/// <inheritdoc />
97+
public int? DepthLimit { get; set; }
98+
99+
/// <inheritdoc />
100+
public bool? DocValues { get; set; }
101+
102+
/// <inheritdoc />
103+
public bool? EagerGlobalOrdinals { get; set; }
104+
105+
/// <inheritdoc />
106+
public int? IgnoreAbove { get; set; }
107+
108+
/// <inheritdoc />
109+
public bool? Index { get; set; }
110+
111+
/// <inheritdoc />
112+
public IndexOptions? IndexOptions { get; set; }
113+
114+
/// <inheritdoc />
115+
public string NullValue { get; set; }
116+
117+
/// <inheritdoc />
118+
public string Similarity { get; set; }
119+
120+
/// <inheritdoc />
121+
public bool? SplitQueriesOnWhitespace { get; set; }
122+
}
123+
124+
/// <inheritdoc cref="IFlattenedProperty" />
125+
[DebuggerDisplay("{DebugDisplay}")]
126+
public class FlattenedPropertyDescriptor<T>
127+
: PropertyDescriptorBase<FlattenedPropertyDescriptor<T>, IFlattenedProperty, T>, IFlattenedProperty
128+
where T : class
129+
{
130+
public FlattenedPropertyDescriptor() : base(FieldType.Flattened) { }
131+
132+
double? IFlattenedProperty.Boost { get; set; }
133+
int? IFlattenedProperty.DepthLimit { get; set; }
134+
bool? IFlattenedProperty.DocValues { get; set; }
135+
bool? IFlattenedProperty.EagerGlobalOrdinals { get; set; }
136+
int? IFlattenedProperty.IgnoreAbove { get; set; }
137+
bool? IFlattenedProperty.Index { get; set; }
138+
IndexOptions? IFlattenedProperty.IndexOptions { get; set; }
139+
string IFlattenedProperty.NullValue { get; set; }
140+
string IFlattenedProperty.Similarity { get; set; }
141+
bool? IFlattenedProperty.SplitQueriesOnWhitespace { get; set; }
142+
143+
/// <inheritdoc cref="IFlattenedProperty.Boost" />
144+
public FlattenedPropertyDescriptor<T> Boost(double? boost) => Assign(boost, (a, v) => a.Boost = v);
145+
146+
/// <inheritdoc cref="IFlattenedProperty.DepthLimit" />
147+
public FlattenedPropertyDescriptor<T> DepthLimit(int? depthLimit) => Assign(depthLimit, (a, v) => a.DepthLimit = v);
148+
149+
/// <inheritdoc cref="IFlattenedProperty.DepthLimit" />
150+
public FlattenedPropertyDescriptor<T> DocValues(bool? docValues = true) => Assign(docValues, (a, v) => a.DocValues = v);
151+
152+
/// <inheritdoc cref="IFlattenedProperty.EagerGlobalOrdinals" />
153+
public FlattenedPropertyDescriptor<T> EagerGlobalOrdinals(bool? eagerGlobalOrdinals = true) =>
154+
Assign(eagerGlobalOrdinals, (a, v) => a.EagerGlobalOrdinals = v);
155+
156+
/// <inheritdoc cref="IFlattenedProperty.IgnoreAbove" />
157+
public FlattenedPropertyDescriptor<T> IgnoreAbove(int? ignoreAbove) => Assign(ignoreAbove, (a, v) => a.IgnoreAbove = v);
158+
159+
/// <inheritdoc cref="IFlattenedProperty.Index" />
160+
public FlattenedPropertyDescriptor<T> Index(bool? index = true) => Assign(index, (a, v) => a.Index = v);
161+
162+
/// <inheritdoc cref="IFlattenedProperty.IndexOptions" />
163+
public FlattenedPropertyDescriptor<T> IndexOptions(IndexOptions? indexOptions) => Assign(indexOptions, (a, v) => a.IndexOptions = v);
164+
165+
/// <inheritdoc cref="IFlattenedProperty.NullValue" />
166+
public FlattenedPropertyDescriptor<T> NullValue(string nullValue) => Assign(nullValue, (a, v) => a.NullValue = v);
167+
168+
/// <inheritdoc cref="IFlattenedProperty.SplitQueriesOnWhitespace" />
169+
public FlattenedPropertyDescriptor<T> SplitQueriesOnWhitespace(bool? splitQueriesOnWhitespace = true) =>
170+
Assign(splitQueriesOnWhitespace, (a, v) => a.SplitQueriesOnWhitespace = v);
171+
172+
/// <inheritdoc cref="IFlattenedProperty.Similarity" />
173+
public FlattenedPropertyDescriptor<T> Similarity(string similarity) => Assign(similarity, (a, v) => a.Similarity = v);
174+
}
175+
}

‎src/Nest/Mapping/Types/FieldType.cs

+4-1
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,9 @@ public enum FieldType
132132
RankFeature,
133133

134134
[EnumMember(Value = "rank_features")]
135-
RankFeatures
135+
RankFeatures,
136+
137+
[EnumMember(Value = "flattened")]
138+
Flattened
136139
}
137140
}

‎src/Nest/Mapping/Types/Properties.cs

+3
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,9 @@ public PropertiesDescriptor<T> Object<TChild>(Func<ObjectTypeDescriptor<T, TChil
166166
/// <inheritdoc cref="IRankFeaturesProperty"/>
167167
public PropertiesDescriptor<T> RankFeatures(Func<RankFeaturesPropertyDescriptor<T>, IRankFeaturesProperty> selector) => SetProperty(selector);
168168

169+
/// <inheritdoc cref="IFlattenedProperty"/>
170+
public PropertiesDescriptor<T> Flattened(Func<FlattenedPropertyDescriptor<T>, IFlattenedProperty> selector) => SetProperty(selector);
171+
169172
public PropertiesDescriptor<T> Custom(IProperty customType) => SetProperty(customType);
170173

171174
private PropertiesDescriptor<T> SetProperty<TDescriptor, TInterface>(Func<TDescriptor, TInterface> selector)

‎src/Nest/Mapping/Types/PropertyFormatter.cs

+4
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ public IProperty Deserialize(ref JsonReader reader, IJsonFormatterResolver forma
9191
case FieldType.Alias: return Deserialize<FieldAliasProperty>(ref segmentReader, formatterResolver);
9292
case FieldType.RankFeature: return Deserialize<RankFeatureProperty>(ref segmentReader, formatterResolver);
9393
case FieldType.RankFeatures: return Deserialize<RankFeaturesProperty>(ref segmentReader, formatterResolver);
94+
case FieldType.Flattened: return Deserialize<FlattenedProperty>(ref segmentReader, formatterResolver);
9495
case FieldType.None:
9596
// no "type" field in the property mapping
9697
return Deserialize<ObjectProperty>(ref segmentReader, formatterResolver);
@@ -197,6 +198,9 @@ public void Serialize(ref JsonWriter writer, IProperty value, IJsonFormatterReso
197198
case "rank_features":
198199
Serialize<IRankFeaturesProperty>(ref writer, value, formatterResolver);
199200
break;
201+
case "flattened":
202+
Serialize<IFlattenedProperty>(ref writer, value, formatterResolver);
203+
break;
200204
default:
201205
if (value is IGenericProperty genericProperty)
202206
Serialize<IGenericProperty>(ref writer, genericProperty, formatterResolver);

‎src/Nest/Mapping/Visitor/IMappingVisitor.cs

+4
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ public interface IMappingVisitor
5757
void Visit(IRankFeaturesProperty property);
5858

5959
void Visit(ISearchAsYouTypeProperty property);
60+
61+
void Visit(IFlattenedProperty property);
6062
}
6163

6264
public class NoopMappingVisitor : IMappingVisitor
@@ -116,5 +118,7 @@ public virtual void Visit(IRankFeatureProperty property) { }
116118
public virtual void Visit(IRankFeaturesProperty property) { }
117119

118120
public virtual void Visit(ISearchAsYouTypeProperty property) { }
121+
122+
public virtual void Visit(IFlattenedProperty property) { }
119123
}
120124
}

‎src/Nest/Mapping/Visitor/IPropertyVisitor.cs

+2
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ public interface IPropertyVisitor
5656

5757
void Visit(IProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute);
5858

59+
void Visit(IFlattenedProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute);
60+
5961
IProperty Visit(PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute);
6062

6163
bool SkipProperty(PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute);

‎src/Nest/Mapping/Visitor/MappingWalker.cs

+10-1
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,10 @@ public void Accept(IProperties properties)
223223
});
224224
break;
225225
case FieldType.Join:
226-
Visit<IJoinProperty>(field, t => { _visitor.Visit(t); });
226+
Visit<IJoinProperty>(field, t =>
227+
{
228+
_visitor.Visit(t);
229+
});
227230
break;
228231
case FieldType.RankFeature:
229232
Visit<IRankFeatureProperty>(field, t =>
@@ -237,6 +240,12 @@ public void Accept(IProperties properties)
237240
_visitor.Visit(t);
238241
});
239242
break;
243+
case FieldType.Flattened:
244+
Visit<IFlattenedProperty>(field, t =>
245+
{
246+
_visitor.Visit(t);
247+
});
248+
break;
240249
}
241250
}
242251
}

‎src/Nest/Mapping/Visitor/NoopPropertyVisitor.cs

+2
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,8 @@ public virtual void Visit(ITextProperty type, PropertyInfo propertyInfo, Elastic
5656

5757
public virtual void Visit(IKeywordProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) { }
5858

59+
public virtual void Visit(IFlattenedProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) { }
60+
5961
public virtual IProperty Visit(PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute) => null;
6062

6163
public void Visit(IProperty type, PropertyInfo propertyInfo, ElasticsearchPropertyAttributeBase attribute)

0 commit comments

Comments
 (0)