When reindexing, use the _parent, _routing and _timestamp from the existing document

Updated Reindex integration test to assert these are copied across
Fixes #985
Fixes #1763

Author: russcam

Diff for: src/Benchmarking/project.lock.json

@@ -508,6 +508,7 @@
         "frameworkAssemblies": [
           "System.Diagnostics.Tools",
           "System.IO.Compression.FileSystem",
+          "System.Linq",
           "System.Reflection",
           "System.Runtime",
           "System.Threading.Tasks"
@@ -2716,6 +2717,7 @@
         "frameworkAssemblies": [
           "System.Diagnostics.Tools",
           "System.IO.Compression.FileSystem",
+          "System.Linq",
           "System.Reflection",
           "System.Runtime",
           "System.Threading.Tasks"
@@ -3337,6 +3339,7 @@
         "frameworkAssemblies": [
           "System.Diagnostics.Tools",
           "System.IO.Compression.FileSystem",
+          "System.Linq",
           "System.Reflection",
           "System.Runtime",
           "System.Threading.Tasks"

Diff for: src/Elasticsearch.Net/project.lock.json

@@ -3109,9 +3109,9 @@
   "projectFileDependencyGroups": {
     "": [],
     ".NETFramework,Version=v4.5": [
+      "fx/System ",
       "fx/System.Runtime ",
-      "fx/System.Runtime.Serialization ",
-      "fx/System "
+      "fx/System.Runtime.Serialization "
     ],
     ".NETPlatform,Version=v5.1": [
       "Microsoft.CSharp >= 4.0.1-beta-23225",

Diff for: src/Nest/Document/Multiple/Bulk/BulkOperation/BulkOperationBase.cs

@@ -89,7 +89,7 @@ public abstract class BulkOperationDescriptorBase<TDescriptor, TInterface>
 
 		public TDescriptor Parent(Id parent) => Assign(a => a.Parent = parent);
 
-		public TDescriptor Timestamp(long timestamp) => Assign(a => a.Timestamp = timestamp);
+		public TDescriptor Timestamp(long? timestamp) => Assign(a => a.Timestamp = timestamp);
 
 		public TDescriptor Ttl(Time ttl) => Assign(a => a.Ttl = ttl);
 	}

Diff for: src/Nest/Document/Multiple/Reindex/ReindexObservable.cs

@@ -29,7 +29,6 @@ public IDisposable Subscribe(IObserver<IReindexResponse<T>> observer)
 				observer.OnError(e);
 			}
 			return this;
-
 		}
 
 		private void Reindex(IObserver<IReindexResponse<T>> observer)
@@ -44,7 +43,7 @@ private void Reindex(IObserver<IReindexResponse<T>> observer)
 			var resolvedTo = toIndex.Resolve(this._connectionSettings);
 			resolvedTo.ThrowIfNullOrEmpty(nameof(toIndex));
 
-			var indexSettings = this._client.GetIndexSettings(i=>i.Index(fromIndex));
+			var indexSettings = this._client.GetIndex(fromIndex);
 			Func<CreateIndexDescriptor, ICreateIndexRequest> settings =  (ci) => this._reindexRequest.CreateIndexRequest ?? ci;
 			var createIndexResponse = this._client.CreateIndex(
 				toIndex, (c) => settings(c.InitializeUsing(indexSettings.Indices[resolvedFrom])));
@@ -88,7 +87,22 @@ public IBulkResponse IndexSearchResults(ISearchResponse<T> searchResult,IObserve
 			foreach (var d in searchResult.Hits)
 			{
 				IHit<T> d1 = d;
-				bb.Index<T>(bi => bi.Document(d1.Source).Type(d1.Type).Index(toIndex).Id(d.Id));
+				bb.Index<T>(bi =>
+				{
+					var bid = bi
+						.Document(d1.Source)
+						.Type(d1.Type)
+						.Index(toIndex)
+						.Id(d1.Id)
+						.Routing(d1.Routing)
+						.Timestamp(d1.Timestamp);
+
+					if (d1.Parent != null)
+					{
+						bid.Parent(d1.Parent);
+					}
+					return bid;
+				});
 			}
 
 			var indexResult = this._client.Bulk(b=>bb);
@@ -104,7 +118,6 @@ public IBulkResponse IndexSearchResults(ISearchResponse<T> searchResult,IObserve
 			return indexResult;
 		}
 
-
 		public void Dispose()
 		{
 

Diff for: src/Nest/Search/Search/Hits/Hit.cs

@@ -16,6 +16,7 @@ public interface IHit<out T> where T : class
 		string Id { get; }
 		string Parent { get; }
 		string Routing { get; }
+		long? Timestamp { get; }
 		IEnumerable<object> Sorts { get; }
 		HighlightFieldDictionary Highlights { get; }
 		Explanation Explanation { get; }
@@ -58,6 +59,9 @@ public class Hit<T> : IHit<T>
 		[JsonProperty(PropertyName = "_routing")]
 		public string Routing { get; internal set; }
 
+		[JsonProperty(PropertyName = "_timestamp")]
+		public long? Timestamp { get; internal set; }
+
 		[JsonProperty(PropertyName = "sort")]
 		public IEnumerable<object> Sorts { get; internal set; }
 

Diff for: src/Nest/project.lock.json

@@ -3441,20 +3441,20 @@
       "Newtonsoft.Json >= 8.0.2"
     ],
     ".NETFramework,Version=v4.5": [
+      "fx/System.Dynamic.Runtime ",
       "fx/System.Runtime ",
       "fx/System.Runtime.Serialization ",
-      "fx/System.ServiceModel ",
-      "fx/System.Dynamic.Runtime "
+      "fx/System.ServiceModel "
     ],
     ".NETPlatform,Version=v5.1": [
-      "System.Runtime >= 4.0.21-beta-23225",
-      "System.Linq >= 4.0.0-beta-23109",
-      "System.Linq.Queryable >= 4.0.0-beta-23109",
       "System.Collections >= 4.0.11-beta-23225",
+      "System.Diagnostics.Tools >= 4.0.1-beta-23225",
       "System.Dynamic.Runtime >= 4.0.11-beta-23516",
+      "System.Linq >= 4.0.0-beta-23109",
+      "System.Linq.Queryable >= 4.0.0-beta-23109",
+      "System.Runtime >= 4.0.21-beta-23225",
       "System.Threading >= 4.0.11-beta-23225",
-      "System.Threading.Timer >= 4.0.0-beta-23109",
-      "System.Diagnostics.Tools >= 4.0.1-beta-23225"
+      "System.Threading.Timer >= 4.0.0-beta-23109"
     ]
   }
 }

Diff for: src/Tests/Document/Multiple/Reindex/ReindexApiTests.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Threading;
 using FluentAssertions;
 using Nest;
@@ -7,26 +8,63 @@
 using Tests.Framework.MockData;
 using Xunit;
 using static Nest.Infer;
+using Elasticsearch.Net;
 
 namespace Tests.Document.Multiple.Reindex
 {
 	[CollectionDefinition(IntegrationContext.Reindex)]
-	public class ReindexCluster : ClusterBase, ICollectionFixture<ReindexCluster>, IClassFixture<EndpointUsage> { } 
+	public class ReindexCluster : ClusterBase, ICollectionFixture<ReindexCluster>, IClassFixture<EndpointUsage>
+	{
+		public override void Boostrap()
+		{
+			var seeder = new Seeder(this.Node);
+			seeder.DeleteIndicesAndTemplates();
+			seeder.CreateIndices();
+		}
+	} 
 
 	[Collection(IntegrationContext.Reindex)]
 	public class ReindexApiTests : SerializationTestBase
 	{
 		private readonly IObservable<IReindexResponse<ILazyDocument>> _reindexResult;
 		private readonly IElasticClient _client;
 
-		private static string NewIndexName { get; } = $"dev-copy-{Guid.NewGuid().ToString("N").Substring(8)}";
+		private static string NewIndexName { get; } = $"project-copy-{Guid.NewGuid().ToString("N").Substring(8)}";
+
+		private static string IndexName { get; } = "project";
 
 		public ReindexApiTests(ReindexCluster cluster, EndpointUsage usage)
 		{
 			this._client = cluster.Client();
-			var indexResult = this._client.IndexMany(Developer.Developers);
-			this._client.Refresh(Index<Developer>());
-			this._reindexResult = this._client.Reindex<ILazyDocument>(Index<Developer>(), NewIndexName, r=>r);
+
+			// create a couple of projects
+			var projects = Project.Generator.Generate(2).ToList();
+			var indexProjectsResponse = this._client.IndexMany(projects, IndexName);
+
+			// create a thousand commits and associate with the projects
+			var commits = CommitActivity.Generator.Generate(1000).ToList();
+			var bb = new BulkDescriptor();
+			for (int index = 0; index < commits.Count; index++)
+			{
+				var commit = commits[index];
+				var project = index%2 == 0
+					? projects[0].Name
+					: projects[1].Name;
+
+				bb.Index<CommitActivity>(bi => bi
+					.Document(commit)
+					.Index(IndexName)
+					.Id(commit.Id)
+					.Routing(project)
+					.Parent(project)
+				);
+			}
+
+			var bulkResult = this._client.Bulk(b => bb);
+			bulkResult.IsValid.Should().BeTrue();
+
+			this._client.Refresh(IndexName);
+			this._reindexResult = this._client.Reindex<ILazyDocument>(IndexName, NewIndexName, r=>r);
 		}
 
 		[I] public void ReturnsExpectedResponse()
@@ -37,18 +75,40 @@ [I] public void ReturnsExpectedResponse()
 					completed: () =>
 					{
 						var refresh = this._client.Refresh(NewIndexName);
-						var originalIndexCount = this._client.Count<Developer>();
-						var newIndexCount = this._client.Count<Developer>(c => c.Index(NewIndexName));
+
+						var originalIndexCount = this._client.Count<CommitActivity>(c => c.Index(IndexName));
+						var newIndexCount = this._client.Count<CommitActivity>(c => c.Index(NewIndexName));
 
 						originalIndexCount.Count.Should().BeGreaterThan(0).And.Be(newIndexCount.Count);
 
+						var scroll = "20s";
+
+						var searchResult = this._client.Search<CommitActivity>(s => s
+							.Index(NewIndexName)
+							.From(0)
+							.Size(100)
+							.Query(q => q.MatchAll())
+							.SearchType(SearchType.Scan)
+							.Scroll(scroll)
+						);
+			
+						do
+						{
+							var result = searchResult;
+							searchResult = this._client.Scroll<CommitActivity>(scroll, result.ScrollId);
+							foreach (var hit in searchResult.Hits)
+							{
+								hit.Timestamp.Should().HaveValue();
+								hit.Parent.Should().NotBeNullOrEmpty();
+								hit.Routing.Should().NotBeNullOrEmpty();
+							}
+						} while (searchResult.IsValid && searchResult.Documents.Any());
 						handle.Set();
 					}
 				);
 
 			this._reindexResult.Subscribe(observer);
 			handle.WaitOne(TimeSpan.FromMinutes(3));
-			//await this._reindexResult;
 		}
 	}
 }

Diff for: src/Tests/Framework/Integration/Bootstrappers/Seeder.cs

@@ -113,6 +113,9 @@ private void CreateProjectIndex()
 					.Map<Project>(MapProject)
 					.Map<CommitActivity>(m => m
 						.Parent<Project>()
+						.TimestampField(t => t
+							.Enabled()
+						)
 						.Properties(props => props
 							.Object<Developer>(o => o
 								.Name(p => p.Committer)