Skip to content

Commit e8bc3b9

Browse files
github-actions[bot]Mpdreamz
github-actions[bot]
and
Mpdreamz
authored
Improve test logger and test clusters (#4445) (#4451)
* Small changes, make sure we print errors at the end when running test suite locally. * include branches ending in .x * Move intrusive xpack tests to own cluster (#4446) * Moves tests further into their own clusters * Enrich is now a CoordinatedUsageTest * Slm is in its own cluster * Security is in its own cluster to not have role/users API potentially cluttering the rest of the tests * GetRepository does not need to be an integration test as RepositoryCrud covers this already * Move watcher tests to own cluster * fix failing unit test after moving enrich stats api over to unit test only * update docs Co-authored-by: Martijn Laarman <[email protected]>
1 parent b13129c commit e8bc3b9

File tree

36 files changed

+360
-319
lines changed

36 files changed

+360
-319
lines changed

‎azure-pipelines.yml

+13
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,16 @@
1+
trigger:
2+
batch: true
3+
branches:
4+
include:
5+
- master
6+
- "*.x"
7+
8+
pr:
9+
autoCancel: true
10+
branches:
11+
exclude:
12+
- "backport-*"
13+
114
jobs:
215
- job: StaleDocs
316
pool:

‎build/scripts/scripts.fsproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
<PackageReference Include="FSharp.Core" Version="4.7.0" />
3939

4040
<PackageReference Include="Bullseye" Version="3.1.0" />
41-
<PackageReference Include="Elastic.Managed" Version="0.1.0-ci20191209T135928" />
41+
<PackageReference Include="Elastic.Managed" Version="0.1.0-ci20200224T155731" />
4242

4343
<PackageReference Include="Fake.Core.Environment" Version="5.15.0" />
4444
<PackageReference Include="Fake.Core.SemVer" Version="5.15.0" />

‎tests/.ci.runsettings

+6
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
<RunConfiguration>
33
<CollectSourceInformation>true</CollectSourceInformation>
44
</RunConfiguration>
5+
<LoggerRunSettings>
6+
<Loggers>
7+
<Logger friendlyName="Console" enabled="True"/>
8+
<Logger friendlyName="pretty" enabled="False"/>
9+
</Loggers>
10+
</LoggerRunSettings>
511
<DataCollectionRunSettings>
612
<DataCollectors>
713
<DataCollector friendlyName="XPlat Code Coverage">

‎tests/.runsettings

-10
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,4 @@
1111
<TestRunParameters>
1212
<Parameter name="DisableFullSkipMessages" value="Examples.;"/>
1313
</TestRunParameters>
14-
<DataCollectionRunSettings>
15-
<DataCollectors>
16-
<DataCollector friendlyName="XPlat Code Coverage">
17-
<Configuration>
18-
<Format>cobertura</Format>
19-
<UseSourceLink>true</UseSourceLink>
20-
</Configuration>
21-
</DataCollector>
22-
</DataCollectors>
23-
</DataCollectionRunSettings>
2414
</RunSettings>

‎tests/Tests.Configuration/Tests.Configuration.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,6 @@
33
<TargetFramework>netstandard2.0</TargetFramework>
44
</PropertyGroup>
55
<ItemGroup>
6-
<PackageReference Include="Elastic.Managed" Version="0.1.0-ci20191209T135928" />
6+
<PackageReference Include="Elastic.Managed" Version="0.1.0-ci20200224T155731" />
77
</ItemGroup>
88
</Project>

‎tests/Tests.Core/Extensions/ClientExtensions.cs

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
using Elasticsearch.Net;
2+
using Nest;
3+
4+
namespace Tests.Core.Extensions
5+
{
6+
public static class ClientExtensions
7+
{
8+
public static ClusterHealthResponse WaitForSecurityIndices(this IElasticClient client) =>
9+
client.Cluster.Health(new ClusterHealthRequest(".security-*") { WaitForStatus = WaitForStatus.Green });
10+
}
11+
}

‎tests/Tests.Core/ManagedElasticsearch/Clusters/XPackCluster.cs

-2
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,8 @@ protected virtual ConnectionSettings ConnectionSettings(ConnectionSettings s) =>
5050
protected sealed override void SeedCluster()
5151
{
5252
Client.Cluster.Health(new ClusterHealthRequest { WaitForStatus = WaitForStatus.Green });
53-
Client.Cluster.Health(new ClusterHealthRequest(".security-7") { WaitForStatus = WaitForStatus.Green });
5453
SeedNode();
5554
Client.Cluster.Health(new ClusterHealthRequest { WaitForStatus = WaitForStatus.Green });
56-
Client.Cluster.Health(new ClusterHealthRequest(".security-7") { WaitForStatus = WaitForStatus.Green });
5755
}
5856

5957
protected virtual void SeedNode() => new DefaultSeeder(Client).SeedNode();

‎tests/Tests.Core/Tests.Core.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
<PackageReference Include="xunit" Version="2.3.1" />
1919
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.1.1" />
2020
<PackageReference Include="coverlet.collector" Version="1.1.0" />
21-
<PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20191209T135928" />
21+
<PackageReference Include="Elastic.Xunit" Version="0.1.0-ci20200224T155731" />
2222
<PackageReference Include="FluentAssertions" Version="5.7.0" />
2323

2424
<PackageReference Include="DiffPlex" Version="1.4.1" />

‎tests/Tests.Domain/Tests.Domain.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
</ItemGroup>
1212
<ItemGroup>
1313
<PackageReference Include="Bogus" Version="22.1.2" />
14-
<PackageReference Include="Elastic.Managed" Version="0.1.0-ci20191209T135928" />
14+
<PackageReference Include="Elastic.Managed" Version="0.1.0-ci20200224T155731" />
1515
<PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
1616
<ProjectReference Include="$(SolutionRoot)\tests\Tests.Configuration\Tests.Configuration.csproj" />
1717
</ItemGroup>

‎tests/Tests.TestLogger/PrettyLogger.cs

+90-63
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Collections.Concurrent;
23
using System.Collections.Generic;
34
using System.Diagnostics;
45
using System.Globalization;
@@ -25,9 +26,17 @@ public class PrettyLogger : ITestLogger
2526
private readonly List<string> _disableSkipNamespaces = new List<string>();
2627
public static Uri RootUri { get; } = new Uri(Environment.CurrentDirectory + Path.DirectorySeparatorChar, UriKind.Absolute);
2728

29+
private readonly ConcurrentQueue<TestResult> _failedTests = new ConcurrentQueue<TestResult>();
30+
2831
public void Initialize(TestLoggerEvents events, string testRunDirectory)
2932
{
30-
events.TestResult += TestResultHandler;
33+
events.TestResult += (s, e) =>
34+
{
35+
if (e.Result.Outcome != TestOutcome.Failed) return;
36+
_failedTests.Enqueue(e.Result);
37+
};
38+
39+
//events.TestResult += TestResultHandler;
3140
events.TestRunComplete += TestRunCompleteHandler;
3241
events.TestRunStart += (sender, args) =>
3342
{
@@ -49,7 +58,6 @@ public void Initialize(TestLoggerEvents events, string testRunDirectory)
4958
.Where(s => !string.IsNullOrWhiteSpace(s))
5059
);
5160
}
52-
foreach (var a in StartUpActions) a();
5361
};
5462
}
5563

@@ -77,37 +85,44 @@ public void TestResultHandler(object sender, TestResultEventArgs e)
7785

7886
break;
7987
default:
80-
if (_writtenPassed > 0)
81-
{
82-
Console.WriteLine();
83-
_writtenPassed = 0;
84-
}
85-
PrintTestOutcomeHeader(e.Result.Outcome, testCase.FullyQualifiedName);
86-
switch (e.Result.Outcome)
87-
{
88-
case TestOutcome.NotFound: break;
89-
case TestOutcome.None: break;
90-
case TestOutcome.Passed:
91-
PrintLocation(testCase);
92-
PrintDuration(e.Result.Duration);
93-
break;
94-
case TestOutcome.Skipped:
95-
foreach (var p in e.Result.Messages)
96-
p.Text.WriteWordWrapped();
97-
98-
break;
99-
case TestOutcome.Failed:
100-
PrintLocation(testCase);
101-
PrintDuration(e.Result.Duration);
102-
e.Result.ErrorMessage.WriteWordWrapped(WordWrapper.WriteWithExceptionHighlighted);
103-
PrintStackTrace(e.Result.ErrorStackTrace);
104-
break;
105-
}
88+
WriteTestResult(e.Result);
10689

10790
break;
10891
}
10992
}
11093

94+
private void WriteTestResult(TestResult result, bool longForm = true)
95+
{
96+
if (_writtenPassed > 0)
97+
{
98+
Console.WriteLine();
99+
_writtenPassed = 0;
100+
}
101+
var testCase = result.TestCase;
102+
PrintTestOutcomeHeader(result.Outcome, result.TestCase.FullyQualifiedName);
103+
switch (result.Outcome)
104+
{
105+
case TestOutcome.NotFound: break;
106+
case TestOutcome.None: break;
107+
case TestOutcome.Passed:
108+
PrintLocation(testCase);
109+
PrintDuration(result.Duration);
110+
break;
111+
case TestOutcome.Skipped:
112+
foreach (var p in result.Messages)
113+
p.Text.WriteWordWrapped();
114+
115+
break;
116+
case TestOutcome.Failed:
117+
PrintLocation(testCase);
118+
PrintDuration(result.Duration);
119+
result.ErrorMessage.WriteWordWrapped(WordWrapper.WriteWithExceptionHighlighted, longForm);
120+
if (longForm)
121+
PrintStackTrace(result.ErrorStackTrace);
122+
break;
123+
}
124+
}
125+
111126
private static int _slowTests = 0;
112127
private static void PrintDuration(TimeSpan duration)
113128
{
@@ -144,20 +159,19 @@ private static void PrintStackTrace(string stackTrace)
144159
var atIn = line.Split(new[] { ") in " }, StringSplitOptions.RemoveEmptyEntries);
145160
var at = atIn[0] + ")";
146161
Console.WriteLine(at);
147-
if (atIn.Length > 1)
148-
{
149-
var @in = atIn[1].Split(':');
150-
var file = @in[0];
151-
var lineNumber = @in[1];
152-
Console.ForegroundColor = ConsoleColor.Gray;
153-
Console.Write(" in ");
154-
Console.ForegroundColor = ConsoleColor.Blue;
155-
Console.Write(lineNumber);
156-
Console.Write(" ");
157-
Console.ForegroundColor = ConsoleColor.DarkGray;
158-
Console.WriteLine(file.CreateRelativePath());
159-
Console.ResetColor();
160-
}
162+
if (atIn.Length <= 1) continue;
163+
164+
var @in = atIn[1].Split(':');
165+
var file = @in[0];
166+
var lineNumber = @in[1];
167+
Console.ForegroundColor = ConsoleColor.Gray;
168+
Console.Write(" in ");
169+
Console.ForegroundColor = ConsoleColor.Blue;
170+
Console.Write(lineNumber);
171+
Console.Write(" ");
172+
Console.ForegroundColor = ConsoleColor.DarkGray;
173+
Console.WriteLine(file.CreateRelativePath());
174+
Console.ResetColor();
161175
}
162176
Console.WriteLine();
163177
}
@@ -172,9 +186,9 @@ private static void PrintLocation(TestCase testCase)
172186
Console.ResetColor();
173187
}
174188

175-
public void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e)
189+
private void TestRunCompleteHandler(object sender, TestRunCompleteEventArgs e)
176190
{
177-
void WriteBox(string boxString, ConsoleColor boxColor, string metric)
191+
static void WriteBox(string boxString, ConsoleColor boxColor, string metric)
178192
{
179193
boxString = " " + boxString.PadRight(5);
180194
Console.ForegroundColor = ConsoleColor.White;
@@ -188,23 +202,17 @@ void WriteBox(string boxString, ConsoleColor boxColor, string metric)
188202
Console.WriteLine();
189203
}
190204

191-
Console.WriteLine();
192-
Console.BackgroundColor = ConsoleColor.White;
193-
Console.ForegroundColor = ConsoleColor.Black;
194-
Console.Write(" ");
195-
Console.ResetColor();
196-
Console.WriteLine();
197-
Console.BackgroundColor = ConsoleColor.White;
198-
Console.ForegroundColor = ConsoleColor.Black;
199-
Console.Write(" 🌈 SUMMARY RESULTS 🌈 ");
200-
Console.ResetColor();
201-
Console.WriteLine();
202-
Console.BackgroundColor = ConsoleColor.White;
203-
Console.ForegroundColor = ConsoleColor.Black;
204-
Console.Write(" ");
205-
Console.ResetColor();
206-
Console.WriteLine();
207-
Console.WriteLine();
205+
206+
//Reprint first 20 test failures at the bottom for convenience
207+
Announce($"SEEN {_failedTests.Count} FAILURE{(_failedTests.Count > 1 ? "S" : "")}");
208+
209+
for (var expanded = 0; _failedTests.TryDequeue(out var testResult); expanded++)
210+
{
211+
WriteTestResult(testResult, expanded <= 20);
212+
}
213+
214+
215+
Announce(" 🌈 SUMMARY RESULTS 🌈 ");
208216

209217
WriteBox("ALL", ConsoleColor.DarkGray, e.TestRunStatistics.ExecutedTests.ToString());
210218

@@ -228,6 +236,27 @@ void WriteBox(string boxString, ConsoleColor boxColor, string metric)
228236

229237
Console.WriteLine();
230238
Console.WriteLine();
239+
}
240+
241+
private static void Announce(string text)
242+
{
243+
Console.WriteLine();
244+
var padding = new string(' ', text.Length + 4);
245+
Console.BackgroundColor = ConsoleColor.White;
246+
Console.ForegroundColor = ConsoleColor.Black;
247+
Console.Write(padding);
248+
Console.ResetColor();
249+
Console.WriteLine();
250+
Console.BackgroundColor = ConsoleColor.White;
251+
Console.ForegroundColor = ConsoleColor.Black;
252+
Console.Write($" {text} ");
253+
Console.ResetColor();
254+
Console.WriteLine();
255+
Console.BackgroundColor = ConsoleColor.White;
256+
Console.ForegroundColor = ConsoleColor.Black;
257+
Console.Write(padding);
258+
Console.ResetColor();
259+
Console.WriteLine();
231260
Console.WriteLine();
232261
}
233262

@@ -303,7 +332,5 @@ private static string ToStringFromMilliseconds(double milliseconds, bool @fixed
303332
return (milliseconds / 60_000d).ToString("N0", Provider) + " m";
304333
}
305334

306-
public static void AddStartupAction(Action action) => StartUpActions.Add(action);
307-
private static List<Action> StartUpActions { get; } = new List<Action>();
308335
}
309336
}

‎tests/Tests.TestLogger/WordWrapper.cs

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,19 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Linq;
34
using System.Text.RegularExpressions;
45

56
namespace Tests.Core.VsTest
67
{
78
internal static class WordWrapper
89
{
9-
public static void WriteWordWrapped(this string paragraph, Action<string> write = null, int tabSize = 4, int indent = 7)
10+
public static void WriteWordWrapped(this string paragraph, Action<string> write = null, bool printAll = true, int tabSize = 4, int indent = 7)
1011
{
1112
write ??= Console.WriteLine;
12-
foreach (var line in paragraph.ToWordWrappedLines(tabSize, indent))
13+
var lines = paragraph.ToWordWrappedLines(tabSize, indent);
14+
if (!printAll)
15+
lines = lines.Take(2).Concat(new[] { $"{new string(' ', indent)} ..abbreviated.." });
16+
foreach (var line in lines)
1317
write(line);
1418
}
1519

‎tests/Tests/Modules/SnapshotAndRestore/Repositories/CleanupRepository/CleanupRepositoryApiTests.cs

-2
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@
77

88
namespace Tests.Modules.SnapshotAndRestore.Repositories.CleanupRepository
99
{
10-
// TODO NOT an integration tests, move SnapshotCrud tests over to a coordinatedtestbase so we can inject this in the request pipeline in order
11-
// todo meaningful assertions
1210
public class CleanupRepositoryApiTests
1311
: ApiTestBase<ReadOnlyCluster, CleanupRepositoryResponse, ICleanupRepositoryRequest, CleanupRepositoryDescriptor, CleanupRepositoryRequest>
1412
{

0 commit comments

Comments
 (0)