Fix CatHelp NodeHotThreads integ tests due to mimetype mismatch (#3883)

* content type can now be dictated on the request

* improved logging for CoordinatedIntegrationTestBase.cs

* SSL warnings on certgen were making xpack tests not run on my windows machine

Author: Mpdreamz

build/scripts/Tooling.fs

@@ -16,7 +16,7 @@ module Tooling =
         let startArgs = StartArguments(bin, args |> List.toArray)
         if (Option.isSome workinDir) then
             startArgs.WorkingDirectory <- Option.defaultValue "" workinDir
-        startArgs.WaitForStreamReadersTimeout <- Nullable<TimeSpan>()
+        if Commandline.isMono then startArgs.WaitForStreamReadersTimeout <- Nullable<TimeSpan>()
         let result = Proc.StartRedirected(startArgs, timeout, LineHighlightWriter())
         if not result.Completed then failwithf "process failed to complete within %O: %s" timeout bin
         if not result.ExitCode.HasValue then failwithf "process yielded no exit code: %s" bin

build/scripts/scripts.fsproj

@@ -69,7 +69,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Bullseye" Version="2.4.0-alpha.1" />
-    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20190621T072151" />
+    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20190626T161237" />
     <PackageReference Include="Fake.Core.Environment" Version="5.15.0" />
     <PackageReference Include="Fake.Core.SemVer" Version="5.15.0" />
     <PackageReference Include="Fake.IO.FileSystem" Version="5.15.0" />

src/Elasticsearch.Net/Connection/HttpWebRequestConnection.cs

@@ -20,6 +20,8 @@ static HttpWebRequestConnection()
 		{
 			//Not available under mono
 			if (!IsMono) HttpWebRequest.DefaultMaximumErrorResponseLength = -1;
+
+
 		}
 
 		internal static bool IsMono { get; } = Type.GetType("Mono.Runtime") != null;
@@ -244,10 +246,13 @@ protected virtual void SetBasicAuthenticationIfNeeded(HttpWebRequest request, Re
 			// 2 - Specified at the global IConnectionSettings level
 			// 3 - Specified with the URI (lowest precedence)
 
-			var userInfo = Uri.UnescapeDataString(requestData.Uri.UserInfo);
+			string userInfo = null;
+			if (!string.IsNullOrEmpty(requestData.Uri.UserInfo))
+				userInfo = Uri.UnescapeDataString(requestData.Uri.UserInfo);
+			else if (requestData.BasicAuthorizationCredentials != null)
+				userInfo =
+					$"{requestData.BasicAuthorizationCredentials.Username}:{requestData.BasicAuthorizationCredentials.Password.CreateString()}";
 
-			if (requestData.BasicAuthorizationCredentials != null)
-				userInfo = requestData.BasicAuthorizationCredentials.ToString();
 
 			if (!string.IsNullOrWhiteSpace(userInfo))
 				request.Headers["Authorization"] = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes(userInfo));

src/Elasticsearch.Net/Transport/Pipeline/RequestData.cs

@@ -12,6 +12,7 @@ namespace Elasticsearch.Net
 	public class RequestData
 	{
 		public const string MimeType = "application/json";
+		public const string MimeTypeTextPlain = "text/plain";
 		public const string OpaqueIdHeader = "X-Opaque-Id";
 		public const string RunAsSecurityHeader = "es-security-runas-user";
 

src/Nest/Cluster/NodesHotThreads/NodesHotThreadsRequest.cs

@@ -1,4 +1,5 @@
-using Elasticsearch.Net.Specification.NodesApi;
+using Elasticsearch.Net;
+using Elasticsearch.Net.Specification.NodesApi;
 
 namespace Nest
 {
@@ -7,12 +8,16 @@ public partial interface INodesHotThreadsRequest { }
 
 	public partial class NodesHotThreadsRequest
 	{
+		protected override string ContentType => RequestData.MimeTypeTextPlain;
+
 		protected sealed override void RequestDefaults(NodesHotThreadsRequestParameters parameters) =>
 			parameters.CustomResponseBuilder = NodeHotThreadsResponseBuilder.Instance;
 	}
 
 	public partial class NodesHotThreadsDescriptor
 	{
+		protected override string ContentType => RequestData.MimeTypeTextPlain;
+		
 		protected sealed override void RequestDefaults(NodesHotThreadsRequestParameters parameters) =>
 			parameters.CustomResponseBuilder = NodeHotThreadsResponseBuilder.Instance;
 	}

src/Nest/CommonAbstractions/Request/RequestBase.cs

@@ -9,6 +9,9 @@ namespace Nest
 	[InterfaceDataContract]
 	public interface IRequest
 	{
+		[IgnoreDataMember]
+		string ContentType { get; }
+
 		[IgnoreDataMember]
 		HttpMethod HttpMethod { get; }
 
@@ -58,6 +61,10 @@ protected RequestBase(Func<RouteValues, RouteValues> pathSelector)
 		[IgnoreDataMember]
 		HttpMethod IRequest.HttpMethod => HttpMethod;
 
+		[IgnoreDataMember]
+		string IRequest.ContentType => ContentType;
+		protected virtual string ContentType { get; } = null;
+
 		private readonly TParameters _parameters;
 
 		[IgnoreDataMember]

src/Nest/ElasticClient.cs

@@ -38,9 +38,11 @@ protected CatResponse<TCatRecord> DoCat<TRequest, TParams, TCatRecord>(TRequest
 			where TRequest : class, IRequest<TParams>
 		{
 			if (typeof(TCatRecord) == typeof(CatHelpRecord))
+			{
 				request.RequestParameters.CustomResponseBuilder = CatHelpResponseBuilder.Instance;
-			else
-				request.RequestParameters.CustomResponseBuilder = CatResponseBuilder<TCatRecord>.Instance;
+				return DoRequest<TRequest, CatResponse<TCatRecord>>(request, request.RequestParameters, r => ElasticClient.ForceTextPlain(r));
+			}
+			request.RequestParameters.CustomResponseBuilder = CatResponseBuilder<TCatRecord>.Instance;
 			return DoRequest<TRequest, CatResponse<TCatRecord>>(request, request.RequestParameters, r => ElasticClient.ForceJson(r));
 		}
 
@@ -50,9 +52,11 @@ protected Task<CatResponse<TCatRecord>> DoCatAsync<TRequest, TParams, TCatRecord
 			where TRequest : class, IRequest<TParams>
 		{
 			if (typeof(TCatRecord) == typeof(CatHelpRecord))
+			{
 				request.RequestParameters.CustomResponseBuilder = CatHelpResponseBuilder.Instance;
-			else
-				request.RequestParameters.CustomResponseBuilder = CatResponseBuilder<TCatRecord>.Instance;
+				return DoRequestAsync<TRequest, CatResponse<TCatRecord>>(request, request.RequestParameters, ct, r => ElasticClient.ForceTextPlain(r));
+			}
+			request.RequestParameters.CustomResponseBuilder = CatResponseBuilder<TCatRecord>.Instance;
 			return DoRequestAsync<TRequest, CatResponse<TCatRecord>>(request, request.RequestParameters, ct, r => ElasticClient.ForceJson(r));
 		}
 
@@ -103,6 +107,7 @@ internal TResponse DoRequest<TRequest, TResponse>(TRequest p, IRequestParameters
 			where TResponse : class, IElasticsearchResponse, new()
 		{
 			if (forceConfiguration != null) ForceConfiguration(p, forceConfiguration);
+			if (p.ContentType != null) ForceContentType(p, p.ContentType);
 
 			var url = p.GetUrl(ConnectionSettings);
 			var b = (p.HttpMethod == HttpMethod.GET || p.HttpMethod == HttpMethod.HEAD) ? null : new SerializableData<TRequest>(p);
@@ -120,6 +125,7 @@ internal Task<TResponse> DoRequestAsync<TRequest, TResponse>(
 			where TResponse : class, IElasticsearchResponse, new()
 		{
 			if (forceConfiguration != null) ForceConfiguration(p, forceConfiguration);
+			if (p.ContentType != null) ForceContentType(p, p.ContentType);
 
 			var url = p.GetUrl(ConnectionSettings);
 			var b = (p.HttpMethod == HttpMethod.GET || p.HttpMethod == HttpMethod.HEAD) ? null : new SerializableData<TRequest>(p);
@@ -130,16 +136,29 @@ internal Task<TResponse> DoRequestAsync<TRequest, TResponse>(
 		private static void ForceConfiguration(IRequest request, Action<IRequestConfiguration> forceConfiguration)
 		{
 			if (forceConfiguration == null) return;
+			
 			var configuration = request.RequestParameters.RequestConfiguration ?? new RequestConfiguration();
 			forceConfiguration(configuration);
 			request.RequestParameters.RequestConfiguration = configuration;
 		}
+		private void ForceContentType<TRequest>(TRequest request, string contentType) where TRequest : class, IRequest
+		{
+			var configuration = request.RequestParameters.RequestConfiguration ?? new RequestConfiguration();
+			configuration.Accept = contentType;
+			configuration.ContentType = contentType;
+			request.RequestParameters.RequestConfiguration = configuration;
+		}
 
 		internal static void ForceJson(IRequestConfiguration requestConfiguration)
 		{
 			requestConfiguration.Accept = RequestData.MimeType;
 			requestConfiguration.ContentType = RequestData.MimeType;
 		}
+		internal static void ForceTextPlain(IRequestConfiguration requestConfiguration)
+		{
+			requestConfiguration.Accept = RequestData.MimeTypeTextPlain;
+			requestConfiguration.ContentType = RequestData.MimeTypeTextPlain;
+		}
 
 		internal IRequestParameters ResponseBuilder(SourceRequestParameters parameters, CustomResponseBuilderBase builder)
 		{

src/Tests/Tests.Benchmarking/Tests.Benchmarking.csproj

@@ -13,7 +13,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
-    <PackageReference Include="Elastic.BenchmarkDotNetExporter" Version="0.1.0-ci20190621T072151" />
+    <PackageReference Include="Elastic.BenchmarkDotNetExporter" Version="0.1.0-ci20190626T161237" />
     <PackageReference Include="LibGit2Sharp" Version="0.26.0-preview-0062" />
   </ItemGroup>
 </Project>

src/Tests/Tests.Core/Tests.Core.csproj

@@ -13,10 +13,10 @@
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Tests.Domain\Tests.Domain.csproj" />
-    <PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20190621T072151" />
+    <PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20190626T161237" />
     <PackageReference Include="Proc" Version="0.6.1" />
     <PackageReference Include="xunit" Version="2.3.1" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.1.1" />
     <PackageReference Include="FluentAssertions" Version="5.7.0" />
     <PackageReference Include="DiffPlex" Version="1.4.1" />
   </ItemGroup>

src/Tests/Tests.Domain/Tests.Domain.csproj

@@ -12,7 +12,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Bogus" Version="22.1.2" />
-    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20190621T072151" />
+    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20190626T161237" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
     <ProjectReference Include="..\Tests.Configuration\Tests.Configuration.csproj" />
   </ItemGroup>

src/Tests/Tests/ClientConcepts/Troubleshooting/DiagnosticSource.doc.cs

@@ -26,27 +26,27 @@ namespace Tests.ClientConcepts.Troubleshooting
 	 *
 	 * To aid with their discover the topics you can subscribe on and the event names they emit are exposed as
 	 * strongly typed strings under `Elasticsearch.Net.Diagnostics.DiagnosticSources`
-	 * 
+	 *
 	 */
 	public class DiagnosticSourceUsageDocumentation : IClusterFixture<ReadOnlyCluster>
 	{
 		private readonly ReadOnlyCluster _cluster;
 
 		public DiagnosticSourceUsageDocumentation(ReadOnlyCluster cluster) => _cluster = cluster;
-		
-		
+
+
 		/**
 		 * Subscribing to DiagnosticSources means implementing `IObserver<DiagnosticListener>`
 		 * or use `.Subscribe(observer, filter)` to opt in to the correct topic.
 		 *
 		 * Here we choose the more verbose `IObserver<>` implementation.
-		 * 
+		 *
 		 */
 		private class ListenerObserver : IObserver<DiagnosticListener>, IDisposable
 		{
 			private long _messagesWrittenToConsole = 0;
 			public long MessagesWrittenToConsole => _messagesWrittenToConsole;
-			
+
 			public Exception SeenException { get; private set; }
 			public void OnError(Exception error) => SeenException = error;
 
@@ -56,12 +56,11 @@ private class ListenerObserver : IObserver<DiagnosticListener>, IDisposable
 			private void WriteToConsole<T>(string eventName, T data)
 			{
 				var a = Activity.Current;
-				Console.WriteLine($"{eventName?.PadRight(30)} {a.Id?.PadRight(32)} {a.ParentId?.PadRight(32)} {data?.ToString().PadRight(10)}");
 				Interlocked.Increment(ref _messagesWrittenToConsole);
 			}
-			
+
 			private List<IDisposable> Disposables { get; } = new List<IDisposable>();
-			
+
 			/**
 			 * By inspecting the name we selectively subscribe only to topics `Elasticsearch.Net` emits.
 			 *
@@ -72,7 +71,7 @@ private void WriteToConsole<T>(string eventName, T data)
 			 *
 			 * Therefor each topic we ship with has a dedicated `Observer` implementation that takes an `onNext` lambda
 			 * which is typed to the context object we actually emit.
-			 * 
+			 *
 			 */
 			public void OnNext(DiagnosticListener value)
 			{
@@ -83,24 +82,24 @@ void TrySubscribe(string sourceName, Func<IObserver<KeyValuePair<string, object>
 					var subscription = value.Subscribe(listener());
 					Disposables.Add(subscription);
 				}
-				
-				TrySubscribe(DiagnosticSources.AuditTrailEvents.SourceName, 
+
+				TrySubscribe(DiagnosticSources.AuditTrailEvents.SourceName,
 					() => new AuditDiagnosticObserver(v => WriteToConsole(v.EventName, v.Audit)));
-				
-				TrySubscribe(DiagnosticSources.Serializer.SourceName, 
+
+				TrySubscribe(DiagnosticSources.Serializer.SourceName,
 					() => new SerializerDiagnosticObserver(v => WriteToConsole(v.EventName, v.Registration)));
 				/**
 				 * RequestPipeline emits a different context object for the start of the `Activity` then it does
 				 * for the end of the `Activity` therefor `RequestPipelineDiagnosticObserver` accepts two `onNext` lambda's.
 				 * One for the `.Start` events and one for the `.Stop` events.
 				 */
-				TrySubscribe(DiagnosticSources.RequestPipeline.SourceName, 
+				TrySubscribe(DiagnosticSources.RequestPipeline.SourceName,
 					() => new RequestPipelineDiagnosticObserver(
 						v => WriteToConsole(v.EventName, v.RequestData),
 						v => WriteToConsole(v.EventName, v.Response)
 					));
-				
-				TrySubscribe(DiagnosticSources.HttpConnection.SourceName, 
+
+				TrySubscribe(DiagnosticSources.HttpConnection.SourceName,
 					() => new HttpConnectionDiagnosticObserver(
 						v => WriteToConsole(v.EventName, v.RequestData),
 						v => WriteToConsole(v.EventName, v.StatusCode)
@@ -122,7 +121,7 @@ [I] public void SubscribeToTopics()
 			using(var listenerObserver = new ListenerObserver())
 			using (var subscription = DiagnosticListener.AllListeners.Subscribe(listenerObserver))
 			{
-				
+
 				/**
 				 * We'll use a Sniffing connection pool here since it sniffs on startup and pings before
 				 * first usage, so our diagnostics are involved enough to showcase most topics.

src/Tests/Tests/Framework/EndpointTests/CoordinatedIntegrationTestBase.cs

@@ -1,8 +1,12 @@
 using System;
+using System.Diagnostics;
+using System.Runtime.ExceptionServices;
 using System.Threading.Tasks;
 using Elastic.Managed.Ephemeral;
 using Elastic.Xunit.XunitPlumbing;
+using Elasticsearch.Net;
 using Nest;
+using Tests.Core.Client;
 using Tests.Core.ManagedElasticsearch.Clusters;
 using Tests.Framework.EndpointTests.TestState;
 using Tests.Framework.Extensions;
@@ -53,7 +57,23 @@ private async Task AssertOnAllResponses<TResponse>(string name, LazyResponses re
 				if (!_coordinatedUsage.MethodIsolatedValues.TryGetValue(key, out var isolatedValue))
 					throw new Exception($"{name} is not a request observed and so no call isolated values could be located for it");
 
-				assert(isolatedValue, response);
+				var r = response;
+				if (TestClient.Configuration.RunIntegrationTests && !r.IsValid && r.ApiCall.OriginalException != null
+					&& !(r.ApiCall.OriginalException is ElasticsearchClientException))
+				{
+					var e = ExceptionDispatchInfo.Capture(r.ApiCall.OriginalException.Demystify());
+					throw new ResponseAssertionException(e.SourceException, r).Demystify();
+				}
+
+				try
+				{
+					assert(isolatedValue, response);
+				}
+				catch (Exception e)
+				{
+					var ex = ExceptionDispatchInfo.Capture(e.Demystify());
+					throw new ResponseAssertionException(ex.SourceException, r).Demystify();
+				}
 			}
 		}
 

src/Tests/Tests/Indices/IndexManagement/RolloverIndex/RolloverIndexApiTests.cs

@@ -53,9 +53,9 @@ public RolloverIndexApiTests(WritableCluster cluster, EndpointUsage usage)
 					}
 				}
 			},
-			aliases = new
+			aliases = new Dictionary<string, object>
 			{
-				new_projects = new { }
+				{ CallIsolatedValue + "-new_projects",  new { } }
 			}
 		};
 
@@ -83,7 +83,7 @@ public RolloverIndexApiTests(WritableCluster cluster, EndpointUsage usage)
 				)
 			)
 			.Aliases(a => a
-				.Alias("new_projects")
+				.Alias(CallIsolatedValue + "-new_projects")
 			);
 
 		protected override RolloverIndexRequest Initializer => new RolloverIndexRequest(CallIsolatedValue + "-alias", CallIsolatedValue + "-new")
@@ -120,7 +120,7 @@ public RolloverIndexApiTests(WritableCluster cluster, EndpointUsage usage)
 			},
 			Aliases = new Aliases
 			{
-				{ "new_projects", new Alias() }
+				{ CallIsolatedValue + "-new_projects", new Alias() }
 			}
 		};
 
@@ -138,7 +138,7 @@ protected override void OnBeforeCall(IElasticClient client)
 				.IndexMany(Project.Generator.Generate(1200))
 			);
 			someDocs.ShouldBeValid();
-			
+
 		}
 
 		protected override LazyResponses ClientUsage() => Calls(

src/Tests/Tests/XPack/Security/User/PutUser/PutUserApiTests.cs

@@ -72,7 +72,8 @@ protected override LazyResponses ClientUsage() => Calls(
 
 		protected override PutUserDescriptor NewDescriptor() => new PutUserDescriptor(CallIsolatedValue);
 
-		protected override void ExpectResponse(PutUserResponse response) => response.Created.Should().BeTrue();
+		protected override void ExpectResponse(PutUserResponse response) => 
+			response.Created.Should().BeTrue("{0}", response.DebugInformation);
 	}
 
 	public class PutUserRunAsApiTests : PutUserApiTests