[Backport master] Improve test logger

azure-pipelines.yml

@@ -1,3 +1,16 @@
+trigger:
+  batch: true
+  branches:
+    include:
+    - master
+    - "*.x"
+
+pr:
+  autoCancel: true
+  branches:
+    exclude:
+    - "backport-*" 
+
 jobs:
 - job: StaleDocs
   pool:

build/scripts/scripts.fsproj

@@ -38,7 +38,7 @@
     <PackageReference Include="FSharp.Core" Version="4.7.0" />
 
     <PackageReference Include="Bullseye" Version="3.1.0" />
-    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20191209T135928" />
+    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20200224T155731" />
 
     <PackageReference Include="Fake.Core.Environment" Version="5.15.0" />
     <PackageReference Include="Fake.Core.SemVer" Version="5.15.0" />

tests/.ci.runsettings

@@ -2,6 +2,12 @@
 	<RunConfiguration>
 		<CollectSourceInformation>true</CollectSourceInformation>
 	</RunConfiguration>
+	<LoggerRunSettings>
+		<Loggers>
+			<Logger friendlyName="Console" enabled="True"/>
+			<Logger friendlyName="pretty" enabled="False"/>
+		</Loggers>
+	</LoggerRunSettings>
 	<DataCollectionRunSettings>
 		<DataCollectors>
 			<DataCollector friendlyName="XPlat Code Coverage">

tests/.runsettings

@@ -11,14 +11,4 @@
 	<TestRunParameters>
 		<Parameter name="DisableFullSkipMessages" value="Examples.;"/>
 	</TestRunParameters>
-	<DataCollectionRunSettings>
-		<DataCollectors>
-			<DataCollector friendlyName="XPlat Code Coverage">
-				<Configuration>
-					<Format>cobertura</Format>
-					<UseSourceLink>true</UseSourceLink>
-				</Configuration>
-			</DataCollector>
-		</DataCollectors>
-	</DataCollectionRunSettings>
 </RunSettings>

tests/Tests.Configuration/Tests.Configuration.csproj

@@ -3,6 +3,6 @@
     <TargetFramework>netstandard2.0</TargetFramework>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20191209T135928" />
+    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20200224T155731" />
   </ItemGroup>
 </Project>

tests/Tests.Core/Extensions/ClientExtensions.cs

@@ -0,0 +1,11 @@
+using Elasticsearch.Net;
+using Nest;
+
+namespace Tests.Core.Extensions
+{
+	public static class ClientExtensions
+	{
+		public static ClusterHealthResponse WaitForSecurityIndices(this IElasticClient client) =>
+			client.Cluster.Health(new ClusterHealthRequest(".security-*") { WaitForStatus = WaitForStatus.Green });
+	}
+}

tests/Tests.Core/ManagedElasticsearch/Clusters/XPackCluster.cs

@@ -50,10 +50,8 @@ protected virtual ConnectionSettings ConnectionSettings(ConnectionSettings s) =>
 		protected sealed override void SeedCluster()
 		{
 			Client.Cluster.Health(new ClusterHealthRequest { WaitForStatus = WaitForStatus.Green });
-			Client.Cluster.Health(new ClusterHealthRequest(".security-7") { WaitForStatus = WaitForStatus.Green });
 			SeedNode();
 			Client.Cluster.Health(new ClusterHealthRequest { WaitForStatus = WaitForStatus.Green });
-			Client.Cluster.Health(new ClusterHealthRequest(".security-7") { WaitForStatus = WaitForStatus.Green });
 		}
 
 		protected virtual void SeedNode() => new DefaultSeeder(Client).SeedNode();

tests/Tests.Core/Tests.Core.csproj

@@ -18,7 +18,7 @@
     <PackageReference Include="xunit" Version="2.3.1" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.1.1" />
     <PackageReference Include="coverlet.collector" Version="1.1.0" />
-    <PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20191209T135928" />
+    <PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20200224T155731" />
     <PackageReference Include="FluentAssertions" Version="5.7.0" />
 
     <PackageReference Include="DiffPlex" Version="1.4.1" />

tests/Tests.Domain/Tests.Domain.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Bogus" Version="22.1.2" />
-    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20191209T135928" />
+    <PackageReference Include="Elastic.Managed" Version="0.1.0-ci20200224T155731" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
     <ProjectReference Include="$(SolutionRoot)\tests\Tests.Configuration\Tests.Configuration.csproj" />
   </ItemGroup>

tests/Tests.TestLogger/PrettyLogger.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Globalization;
@@ -25,9 +26,17 @@ public class PrettyLogger : ITestLogger
 		private readonly List<string> _disableSkipNamespaces = new List<string>();
 		public static Uri RootUri { get; } = new Uri(Environment.CurrentDirectory + Path.DirectorySeparatorChar, UriKind.Absolute);
 
+		private readonly ConcurrentQueue<TestResult> _failedTests = new ConcurrentQueue<TestResult>();
+
 		public void Initialize(TestLoggerEvents events, string testRunDirectory)
 		{
-			events.TestResult += TestResultHandler;
+			events.TestResult += (s, e) =>
+			{
+				if (e.Result.Outcome != TestOutcome.Failed) return;
+				_failedTests.Enqueue(e.Result);
+			};
+
+			//events.TestResult += TestResultHandler;
 			events.TestRunComplete += TestRunCompleteHandler;
 			events.TestRunStart += (sender, args) =>
 			{
@@ -49,7 +58,6 @@ public void Initialize(TestLoggerEvents events, string testRunDirectory)
 							.Where(s => !string.IsNullOrWhiteSpace(s))
 						);
 				}
-				foreach (var a in StartUpActions) a();
 			};
 		}
 
@@ -77,37 +85,44 @@ public void TestResultHandler(object sender, TestResultEventArgs e)
 
 					break;
 				default:
-					if (_writtenPassed > 0)
-					{
-						Console.WriteLine();
-						_writtenPassed = 0;
-					}
-					PrintTestOutcomeHeader(e.Result.Outcome, testCase.FullyQualifiedName);
-					switch (e.Result.Outcome)
-					{
-						case TestOutcome.NotFound: break;
-						case TestOutcome.None: break;
-						case TestOutcome.Passed:
-							PrintLocation(testCase);
-							PrintDuration(e.Result.Duration);
-							break;
-						case TestOutcome.Skipped:
-							foreach (var p in e.Result.Messages)
-								p.Text.WriteWordWrapped();
-
-							break;
-						case TestOutcome.Failed:
-							PrintLocation(testCase);
-							PrintDuration(e.Result.Duration);
-							e.Result.ErrorMessage.WriteWordWrapped(WordWrapper.WriteWithExceptionHighlighted);
-							PrintStackTrace(e.Result.ErrorStackTrace);
-							break;
-					}
+					WriteTestResult(e.Result);
 
 					break;
 			}
 		}
 
+		private void WriteTestResult(TestResult result, bool longForm = true)
+		{
+			if (_writtenPassed > 0)
+			{
+				Console.WriteLine();
+				_writtenPassed = 0;
+			}
+			var testCase = result.TestCase;
+			PrintTestOutcomeHeader(result.Outcome, result.TestCase.FullyQualifiedName);
+			switch (result.Outcome)
+			{
+				case TestOutcome.NotFound: break;
+				case TestOutcome.None: break;
+				case TestOutcome.Passed:
+					PrintLocation(testCase);
+					PrintDuration(result.Duration);
+					break;
+				case TestOutcome.Skipped:
+					foreach (var p in result.Messages)
+						p.Text.WriteWordWrapped();
+
+					break;
+				case TestOutcome.Failed:
+					PrintLocation(testCase);
+					PrintDuration(result.Duration);
+					result.ErrorMessage.WriteWordWrapped(WordWrapper.WriteWithExceptionHighlighted, longForm);
+					if (longForm)
+						PrintStackTrace(result.ErrorStackTrace);
+					break;
+			}
+		}
+
 		private static int _slowTests = 0;
 		private static void PrintDuration(TimeSpan duration)
 		{
@@ -144,20 +159,19 @@ private static void PrintStackTrace(string stackTrace)
 				var atIn = line.Split(new[] { ") in " }, StringSplitOptions.RemoveEmptyEntries);
 				var at = atIn[0] + ")";
 				Console.WriteLine(at);
-				if (atIn.Length > 1)
-				{
-					var @in = atIn[1].Split(':');
-					var file = @in[0];
-					var lineNumber = @in[1];
-					Console.ForegroundColor = ConsoleColor.Gray;
-					Console.Write("     in ");
-					Console.ForegroundColor = ConsoleColor.Blue;
-					Console.Write(lineNumber);
-					Console.Write(" ");
-					Console.ForegroundColor = ConsoleColor.DarkGray;
-					Console.WriteLine(file.CreateRelativePath());
-					Console.ResetColor();
-				}
+				if (atIn.Length <= 1) continue;
+
+				var @in = atIn[1].Split(':');
+				var file = @in[0];
+				var lineNumber = @in[1];
+				Console.ForegroundColor = ConsoleColor.Gray;
+				Console.Write("     in ");
+				Console.ForegroundColor = ConsoleColor.Blue;
+				Console.Write(lineNumber);
+				Console.Write(" ");
+				Console.ForegroundColor = ConsoleColor.DarkGray;
+				Console.WriteLine(file.CreateRelativePath());
+				Console.ResetColor();
 			}
 			Console.WriteLine();
 		}
@@ -172,9 +186,9 @@ private static void PrintLocation(TestCase testCase)
 			Console.ResetColor();
 		}
 
-		public void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e)
+		private void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e)
 		{
-			void WriteBox(string boxString, ConsoleColor boxColor, string metric)
+			static void WriteBox(string boxString, ConsoleColor boxColor, string metric)
 			{
 				boxString = " " + boxString.PadRight(5);
 				Console.ForegroundColor = ConsoleColor.White;
@@ -188,23 +202,17 @@ void WriteBox(string boxString, ConsoleColor boxColor, string metric)
 				Console.WriteLine();
 			}
 
-			Console.WriteLine();
-			Console.BackgroundColor = ConsoleColor.White;
-			Console.ForegroundColor = ConsoleColor.Black;
-			Console.Write("                         ");
-			Console.ResetColor();
-			Console.WriteLine();
-			Console.BackgroundColor = ConsoleColor.White;
-			Console.ForegroundColor = ConsoleColor.Black;
-			Console.Write("  🌈 SUMMARY RESULTS 🌈  ");
-			Console.ResetColor();
-			Console.WriteLine();
-			Console.BackgroundColor = ConsoleColor.White;
-			Console.ForegroundColor = ConsoleColor.Black;
-			Console.Write("                         ");
-			Console.ResetColor();
-			Console.WriteLine();
-			Console.WriteLine();
+
+			//Reprint first 20 test failures at the bottom for convenience
+			Announce($"SEEN {_failedTests.Count} FAILURE{(_failedTests.Count > 1 ? "S" : "")}");
+
+			for (var expanded = 0; _failedTests.TryDequeue(out var testResult); expanded++)
+			{
+				WriteTestResult(testResult, expanded <= 20);
+			}
+
+
+			Announce(" 🌈 SUMMARY RESULTS 🌈 ");
 
 			WriteBox("ALL", ConsoleColor.DarkGray, e.TestRunStatistics.ExecutedTests.ToString());
 
@@ -228,6 +236,27 @@ void WriteBox(string boxString, ConsoleColor boxColor, string metric)
 
 			Console.WriteLine();
 			Console.WriteLine();
+		}
+
+		private static void Announce(string text)
+		{
+			Console.WriteLine();
+			var padding = new string(' ', text.Length + 4);
+			Console.BackgroundColor = ConsoleColor.White;
+			Console.ForegroundColor = ConsoleColor.Black;
+			Console.Write(padding);
+			Console.ResetColor();
+			Console.WriteLine();
+			Console.BackgroundColor = ConsoleColor.White;
+			Console.ForegroundColor = ConsoleColor.Black;
+			Console.Write($"  {text}  ");
+			Console.ResetColor();
+			Console.WriteLine();
+			Console.BackgroundColor = ConsoleColor.White;
+			Console.ForegroundColor = ConsoleColor.Black;
+			Console.Write(padding);
+			Console.ResetColor();
+			Console.WriteLine();
 			Console.WriteLine();
 		}
 
@@ -303,7 +332,5 @@ private static string ToStringFromMilliseconds(double milliseconds, bool @fixed
 			return (milliseconds / 60_000d).ToString("N0", Provider) + " m";
 		}
 
-		public static void AddStartupAction(Action action) => StartUpActions.Add(action);
-		private static List<Action> StartUpActions { get; } = new List<Action>();
 	}
 }

tests/Tests.TestLogger/WordWrapper.cs

@@ -1,15 +1,19 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text.RegularExpressions;
 
 namespace Tests.Core.VsTest
 {
 	internal static class WordWrapper
 	{
-		public static void WriteWordWrapped(this string paragraph, Action<string> write = null, int tabSize = 4, int indent = 7)
+		public static void WriteWordWrapped(this string paragraph, Action<string> write = null, bool printAll = true, int tabSize = 4, int indent = 7)
 		{
 			write ??= Console.WriteLine;
-			foreach (var line in paragraph.ToWordWrappedLines(tabSize, indent))
+			var lines = paragraph.ToWordWrappedLines(tabSize, indent);
+			if (!printAll)
+				lines = lines.Take(2).Concat(new[] { $"{new string(' ', indent)} ..abbreviated.." });
+			foreach (var line in lines)
 				write(line);
 		}
 

tests/Tests/Modules/SnapshotAndRestore/Repositories/CleanupRepository/CleanupRepositoryApiTests.cs

@@ -7,8 +7,6 @@
 
 namespace Tests.Modules.SnapshotAndRestore.Repositories.CleanupRepository
 {
-	// TODO NOT an integration tests, move SnapshotCrud tests over to a coordinatedtestbase so we can inject this in the request pipeline in order
-	// todo meaningful assertions
 	public class CleanupRepositoryApiTests
 		: ApiTestBase<ReadOnlyCluster, CleanupRepositoryResponse, ICleanupRepositoryRequest, CleanupRepositoryDescriptor, CleanupRepositoryRequest>
 	{