Skip to content

Commit 163dc20

Browse files
russcamrusscam
authored
Return distinct items from GetMany and SourceMany (#4353) (#4400)
This commit fixes a bug where a GetMany or SourceMany API call with a repeated id would return a cartesian product of id and documents. - enumerate only distinct ids when retrieving hits or source from GetMany and SourceMany, so that the same id input to either will return only a single document per target index. - fix a bug in the MultiGetRequestFormatter whereby the document index is removed when a request index is specified, without checking whether the document index matches the request index. Fixes #4342 (cherry-picked from commit 8cbc1fe)
1 parent beacf89 commit 163dc20

File tree

8 files changed

+232
-54
lines changed

8 files changed

+232
-54
lines changed

‎docs/aggregations/bucket/terms/terms-aggregation-usage.asciidoc

+2-2
Original file line numberDiff line numberDiff line change
@@ -206,15 +206,15 @@ new TermsAggregation("states")
206206
[source,csharp]
207207
----
208208
response.ShouldBeValid();
209-
var states = response.Aggregations.Terms("states");
209+
var states = response.Aggregations.Terms<StateOfBeing>("states");
210210
states.Should().NotBeNull();
211211
states.DocCountErrorUpperBound.Should().HaveValue();
212212
states.SumOtherDocCount.Should().HaveValue();
213213
states.Buckets.Should().NotBeNull();
214214
states.Buckets.Count.Should().BeGreaterThan(0);
215215
foreach (var item in states.Buckets)
216216
{
217-
item.Key.Should().NotBeNullOrEmpty();
217+
item.Key.Should().BeOfType<StateOfBeing>();
218218
item.DocCount.Should().BeGreaterOrEqualTo(1);
219219
}
220220
states.Meta.Should().NotBeNull().And.HaveCount(1);

‎docs/search/request/suggest-usage.asciidoc

+9
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ See the Elasticsearch documentation on {ref_current}/search-suggesters.html[Sugg
2626
----
2727
s => s
2828
.Query(q => ProjectFilter)
29+
.DocValueFields(d => d
30+
.Field(f => f.State)
31+
)
2932
.Suggest(ss => ss
3033
.Term("my-term-suggest", t => t
3134
.MaxEdits(1)
@@ -89,6 +92,7 @@ s => s
8992
new SearchRequest<Project>
9093
{
9194
Query = ProjectFilter,
95+
DocValueFields = Fields<Project>(f => f.State),
9296
Suggest = new SuggestContainer
9397
{
9498
{
@@ -185,6 +189,9 @@ new SearchRequest<Project>
185189
}
186190
}
187191
},
192+
"docvalue_fields": [
193+
"state"
194+
],
188195
"suggest": {
189196
"my-completion-suggest": {
190197
"completion": {
@@ -273,6 +280,8 @@ option.Source.Should().NotBeNull();
273280
option.Source.Name.Should().NotBeNullOrWhiteSpace();
274281
option.Source.ShouldAdhereToSourceSerializerWhenSet();
275282
option.Score.Should().BeGreaterThan(0);
283+
option.Fields.Should().NotBeNull().And.NotBeEmpty();
284+
option.Fields.Should().ContainKey("state");
276285
option.Contexts.Should().NotBeNull().And.NotBeEmpty();
277286
option.Contexts.Should().ContainKey("color");
278287
var colorContexts = option.Contexts["color"];

‎docs/search/writing-queries.asciidoc

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ please modify the original csharp file found at the link and submit the PR with
1616
=== Writing queries
1717

1818
Once you have data indexed within Elasticsearch, you're going to want to be able to search it. Elasticsearch
19-
offers a powerful query DSL to define queries to execute agains Elasticsearch. This DSL is based on JSON
19+
offers a powerful query DSL to define queries to execute against Elasticsearch. This DSL is based on JSON
2020
and is exposed in NEST in the form of both a Fluent API and an Object Initializer syntax
2121

2222
==== Match All query

‎src/Nest/Document/Multiple/MultiGet/ElasticClient-GetMany.cs

+8-8
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ public static class GetManyExtensions
2121
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
2222
/// <param name="client"></param>
2323
/// <param name="ids">IEnumerable of ids as string for the documents to fetch</param>
24-
/// <param name="index">Optionally override the default inferred index name for T</param>
25-
/// <param name="type">Optionally overiide the default inferred typename for T</param>
24+
/// <param name="index">Set the request level index name</param>
25+
/// <param name="type">Set the request level type name</param>
2626
public static IEnumerable<IMultiGetHit<T>> GetMany<T>(this IElasticClient client, IEnumerable<string> ids, IndexName index = null,
2727
TypeName type = null
2828
)
@@ -47,8 +47,8 @@ public static IEnumerable<IMultiGetHit<T>> GetMany<T>(this IElasticClient client
4747
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
4848
/// <param name="client"></param>
4949
/// <param name="ids">IEnumerable of ids as ints for the documents to fetch</param>
50-
/// <param name="index">Optionally override the default inferred index name for T</param>
51-
/// <param name="type">Optionally overiide the default inferred typename for T</param>
50+
/// <param name="index">Set the request level index name</param>
51+
/// <param name="type">Set the request level type name</param>
5252
public static IEnumerable<IMultiGetHit<T>> GetMany<T>(this IElasticClient client, IEnumerable<long> ids, IndexName index = null,
5353
TypeName type = null
5454
)
@@ -64,8 +64,8 @@ public static IEnumerable<IMultiGetHit<T>> GetMany<T>(this IElasticClient client
6464
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
6565
/// <param name="client"></param>
6666
/// <param name="ids">IEnumerable of ids as string for the documents to fetch</param>
67-
/// <param name="index">Optionally override the default inferred index name for T</param>
68-
/// <param name="type">Optionally overiide the default inferred typename for T</param>
67+
/// <param name="index">Set the request level index name</param>
68+
/// <param name="type">Set the request level type name</param>
6969
public static async Task<IEnumerable<IMultiGetHit<T>>> GetManyAsync<T>(
7070
this IElasticClient client, IEnumerable<string> ids, IndexName index = null, TypeName type = null,
7171
CancellationToken cancellationToken = default(CancellationToken)
@@ -93,8 +93,8 @@ public static async Task<IEnumerable<IMultiGetHit<T>>> GetManyAsync<T>(
9393
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
9494
/// <param name="client"></param>
9595
/// <param name="ids">IEnumerable of ids as ints for the documents to fetch</param>
96-
/// <param name="index">Optionally override the default inferred index name for T</param>
97-
/// <param name="type">Optionally overiide the default inferred typename for T</param>
96+
/// <param name="index">Set the request level index name</param>
97+
/// <param name="type">Set the request level type name</param>
9898
public static Task<IEnumerable<IMultiGetHit<T>>> GetManyAsync<T>(
9999
this IElasticClient client, IEnumerable<long> ids, IndexName index = null, TypeName type = null,
100100
CancellationToken cancellationToken = default(CancellationToken)

‎src/Nest/Document/Multiple/MultiGet/ElasticClient-SourceMany.cs

+14-16
Original file line numberDiff line numberDiff line change
@@ -24,17 +24,16 @@ public static class SourceManyExtensions
2424
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
2525
/// <param name="client"></param>
2626
/// <param name="ids">A list of ids as string</param>
27-
/// <param name="index">Optionally override the default inferred indexname for T</param>
28-
/// <param name="type">Optionally override the default inferred indexname for T</param>
27+
/// <param name="index">Set the request level index name</param>
28+
/// <param name="type">Set the request level type name</param>
2929
public static IEnumerable<T> SourceMany<T>(this IElasticClient client, IEnumerable<string> ids, string index = null, string type = null)
3030
where T : class
3131
{
3232
var result = client.MultiGet(s => s
33+
.Index(index)
34+
.Type(type)
3335
.RequestConfiguration(r => r.ThrowExceptions())
34-
.GetMany<T>(ids, (gs, i) => gs
35-
.Index(index)
36-
.Type(type)
37-
)
36+
.GetMany<T>(ids)
3837
);
3938
return result.SourceMany<T>(ids);
4039
}
@@ -52,8 +51,8 @@ public static IEnumerable<T> SourceMany<T>(this IElasticClient client, IEnumerab
5251
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
5352
/// <param name="client"></param>
5453
/// <param name="ids">A list of ids as int</param>
55-
/// <param name="index">Optionally override the default inferred indexname for T</param>
56-
/// <param name="type">Optionally override the default inferred indexname for T</param>
54+
/// <param name="index">Set the request level index name</param>
55+
/// <param name="type">Set the request level type name</param>
5756
public static IEnumerable<T> SourceMany<T>(this IElasticClient client, IEnumerable<long> ids, string index = null, string type = null)
5857
where T : class => client.SourceMany<T>(ids.Select(i => i.ToString(CultureInfo.InvariantCulture)), index, type);
5958

@@ -70,20 +69,19 @@ public static IEnumerable<T> SourceMany<T>(this IElasticClient client, IEnumerab
7069
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
7170
/// <param name="client"></param>
7271
/// <param name="ids">A list of ids as string</param>
73-
/// <param name="index">Optionally override the default inferred indexname for T</param>
74-
/// <param name="type">Optionally override the default inferred indexname for T</param>
72+
/// <param name="index">Set the request level index name</param>
73+
/// <param name="type">Set the request level type name</param>
7574
public static async Task<IEnumerable<T>> SourceManyAsync<T>(
7675
this IElasticClient client, IEnumerable<string> ids, string index = null, string type = null,
7776
CancellationToken cancellationToken = default(CancellationToken)
7877
)
7978
where T : class
8079
{
8180
var response = await client.MultiGetAsync(s => s
81+
.Index(index)
82+
.Type(type)
8283
.RequestConfiguration(r => r.ThrowExceptions())
83-
.GetMany<T>(ids, (gs, i) => gs
84-
.Index(index)
85-
.Type(type)
86-
), cancellationToken)
84+
.GetMany<T>(ids), cancellationToken)
8785
.ConfigureAwait(false);
8886
return response.SourceMany<T>(ids);
8987
}
@@ -101,8 +99,8 @@ public static async Task<IEnumerable<T>> SourceManyAsync<T>(
10199
/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
102100
/// <param name="client"></param>
103101
/// <param name="ids">A list of ids as int</param>
104-
/// <param name="index">Optionally override the default inferred indexname for T</param>
105-
/// <param name="type">Optionally override the default inferred indexname for T</param>
102+
/// <param name="index">Set the request level index name</param>
103+
/// <param name="type">Set the request level type name</param>
106104
public static Task<IEnumerable<T>> SourceManyAsync<T>(
107105
this IElasticClient client, IEnumerable<long> ids, string index = null, string type = null,
108106
CancellationToken cancellationToken = default(CancellationToken)

‎src/Nest/Document/Multiple/MultiGet/Request/MultiGetRequestJsonConverter.cs

+40-8
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
using System;
2+
using System.Collections.Generic;
23
using System.Linq;
4+
using Elasticsearch.Net;
35
using Newtonsoft.Json;
46

57
namespace Nest
@@ -13,20 +15,50 @@ internal class MultiGetRequestJsonConverter : JsonConverter
1315

1416
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
1517
{
16-
var request = value as IMultiGetRequest;
18+
var request = (IMultiGetRequest)value;
1719
writer.WriteStartObject();
1820
if (!(request?.Documents.HasAny()).GetValueOrDefault(false))
1921
{
2022
writer.WriteEndObject();
2123
return;
2224
}
23-
var docs = request.Documents.Select(d =>
24-
{
25-
if (request.Index != null) d.Index = null;
26-
if (request.Type != null) d.Type = null;
27-
return d;
28-
})
29-
.ToList();
25+
26+
List<IMultiGetOperation> docs;
27+
var requestHasIndex = request.Index != null;
28+
var requestHasType = request.Type != null;
29+
30+
if (requestHasIndex || requestHasType)
31+
{
32+
var settings = serializer.GetConnectionSettings();
33+
var resolvedIndex = requestHasIndex
34+
? request.Index.GetString(settings)
35+
: null;
36+
var resolvedType = requestHasType
37+
? ((IUrlParameter)request.Type).GetString(settings)
38+
: null;
39+
40+
docs = request.Documents.Select(d =>
41+
{
42+
if (requestHasIndex && d.Index != null)
43+
{
44+
var docIndex = d.Index.GetString(settings);
45+
if (string.Equals(resolvedIndex, docIndex))
46+
d.Index = null;
47+
}
48+
49+
if (requestHasType && d.Type != null)
50+
{
51+
var docType = ((IUrlParameter)d.Type).GetString(settings);
52+
if (string.Equals(resolvedType, docType))
53+
d.Type = null;
54+
}
55+
56+
return d;
57+
})
58+
.ToList();
59+
}
60+
else
61+
docs = request.Documents.ToList();
3062

3163
var flatten = docs.All(p => p.CanBeFlattened);
3264

‎src/Nest/Document/Multiple/MultiGet/Response/MultiGetResponse.cs

+30-12
Original file line numberDiff line numberDiff line change
@@ -52,12 +52,28 @@ public FieldValues GetFieldValues<T>(string id) where T : class
5252
return multiHit?.Fields ?? FieldValues.Empty;
5353
}
5454

55+
/// <summary>
56+
/// Retrieves the hits for each distinct id.
57+
/// </summary>
58+
/// <param name="ids">The ids to retrieve source for</param>
59+
/// <typeparam name="T">The document type for the hits to return</typeparam>
60+
/// <returns>An IEnumerable{T} of hits</returns>
5561
public IEnumerable<IMultiGetHit<T>> GetMany<T>(IEnumerable<string> ids) where T : class
5662
{
57-
var docs = Hits.OfType<IMultiGetHit<T>>();
58-
return from d in docs
59-
join id in ids on d.Id equals id
60-
select d;
63+
HashSet<string> seenIndices = null;
64+
foreach (var id in ids.Distinct())
65+
{
66+
if (seenIndices == null)
67+
seenIndices = new HashSet<string>();
68+
else
69+
seenIndices.Clear();
70+
71+
foreach (var doc in Hits.OfType<IMultiGetHit<T>>())
72+
{
73+
if (string.Equals(doc.Id, id) && seenIndices.Add(doc.Index))
74+
yield return doc;
75+
}
76+
}
6177
}
6278

6379
public IEnumerable<IMultiGetHit<T>> GetMany<T>(IEnumerable<long> ids) where T : class =>
@@ -71,14 +87,16 @@ public T Source<T>(string id) where T : class
7187

7288
public T Source<T>(long id) where T : class => Source<T>(id.ToString(CultureInfo.InvariantCulture));
7389

74-
public IEnumerable<T> SourceMany<T>(IEnumerable<string> ids) where T : class
75-
{
76-
var docs = Hits.OfType<IMultiGetHit<T>>();
77-
return from d in docs
78-
join id in ids on d.Id equals id
79-
where d.Found
80-
select d.Source;
81-
}
90+
/// <summary>
91+
/// Retrieves the source, if available, for each distinct id.
92+
/// </summary>
93+
/// <param name="ids">The ids to retrieve source for</param>
94+
/// <typeparam name="T">The document type for the hits to return</typeparam>
95+
/// <returns>An IEnumerable{T} of sources</returns>
96+
public IEnumerable<T> SourceMany<T>(IEnumerable<string> ids) where T : class =>
97+
from hit in GetMany<T>(ids)
98+
where hit.Found
99+
select hit.Source;
82100

83101
public IEnumerable<T> SourceMany<T>(IEnumerable<long> ids) where T : class =>
84102
SourceMany<T>(ids.Select(i => i.ToString(CultureInfo.InvariantCulture)));

0 commit comments

Comments
 (0)