Skip to content

Commit 2936272

Browse files
timcasselltimcassell
authored
Add expected results tests (#2361)
* Added manual tests for expected results. * Use a fallback value for cpu resolution. Use ZeroMeasurementHelper. * Add multiple struct sizes. * Use same code as ZeroMeasurementAnalyser. * Use ZeroMeasurementHelper for both checks. * Set benchmark process priority to realtime. * Add threshold argument to ZeroMeasurementHelper. * Friendly benchmark names * Removed RealTimeBenchmarks
1 parent 55a3376 commit 2936272

File tree

5 files changed

+242
-4
lines changed

5 files changed

+242
-4
lines changed

‎src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using Perfolizer.Mathematics.SignificanceTesting;
2+
using Perfolizer.Mathematics.Thresholds;
23

34
namespace BenchmarkDotNet.Analysers
45
{
@@ -19,11 +20,11 @@ public static bool CheckZeroMeasurementOneSample(double[] results, double thresh
1920
/// Checks distribution against Zero Measurement hypothesis in case of two samples
2021
/// </summary>
2122
/// <returns>True if measurement is ZeroMeasurement</returns>
22-
public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead)
23+
public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold threshold = null)
2324
{
2425
if (workload.Length < 3 || overhead.Length < 3)
2526
return false;
26-
return !WelchTest.Instance.IsGreater(workload, overhead).NullHypothesisIsRejected;
27+
return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected;
2728
}
2829
}
2930
}

‎src/BenchmarkDotNet/Properties/AssemblyInfo.cs

+2
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,11 @@
1414
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
1515
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
1616
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
17+
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning,PublicKey=" + BenchmarkDotNetInfo.PublicKey)]
1718
#else
1819
[assembly: InternalsVisibleTo("BenchmarkDotNet.Tests")]
1920
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests")]
2021
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows")]
2122
[assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace")]
23+
[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.ManualRunning")]
2224
#endif

‎tests/BenchmarkDotNet.IntegrationTests.ManualRunning/BenchmarkDotNet.IntegrationTests.ManualRunning.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
<Compile Include="..\BenchmarkDotNet.IntegrationTests\Xunit\MisconfiguredEnvironmentException.cs" Link="MisconfiguredEnvironmentException.cs" />
1717
<Compile Include="..\BenchmarkDotNet.IntegrationTests\Xunit\Extensions.cs" Link="Extensions.cs" />
1818
<Compile Include="..\BenchmarkDotNet.IntegrationTests\TestConfigs.cs" Link="TestConfigs.cs" />
19-
<Compile Include="..\BenchmarkDotNet.Tests\Loggers\OutputLogger.cs" Link="OutputLogger.cs" />
2019
</ItemGroup>
2120
<ItemGroup>
2221
<Content Include="xunit.runner.json">
@@ -26,6 +25,7 @@
2625
<ItemGroup>
2726
<ProjectReference Include="..\..\src\BenchmarkDotNet\BenchmarkDotNet.csproj" />
2827
<ProjectReference Include="..\..\src\BenchmarkDotNet.Diagnostics.Windows\BenchmarkDotNet.Diagnostics.Windows.csproj" />
28+
<ProjectReference Include="..\BenchmarkDotNet.Tests\BenchmarkDotNet.Tests.csproj" />
2929
</ItemGroup>
3030
<ItemGroup>
3131
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />

‎tests/BenchmarkDotNet.IntegrationTests.ManualRunning/ExpectedBenchmarkResultsTests.cs

+235
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Diagnostics;
4+
using System.Linq;
5+
using BenchmarkDotNet.Analysers;
6+
using BenchmarkDotNet.Attributes;
7+
using BenchmarkDotNet.Configs;
8+
using BenchmarkDotNet.Diagnosers;
9+
using BenchmarkDotNet.Engines;
10+
using BenchmarkDotNet.Extensions;
11+
using BenchmarkDotNet.Jobs;
12+
using BenchmarkDotNet.Portability;
13+
using BenchmarkDotNet.Reports;
14+
using BenchmarkDotNet.Tests.XUnit;
15+
using BenchmarkDotNet.Toolchains.InProcess.Emit;
16+
using Perfolizer.Horology;
17+
using Perfolizer.Mathematics.Thresholds;
18+
using Xunit;
19+
using Xunit.Abstractions;
20+
21+
namespace BenchmarkDotNet.IntegrationTests.ManualRunning
22+
{
23+
public class ExpectedBenchmarkResultsTests : BenchmarkTestExecutor
24+
{
25+
// NativeAot takes a long time to build, so not including it in these tests.
26+
// We also don't test InProcessNoEmitToolchain because it is known to be less accurate than code-gen toolchains.
27+
28+
private static readonly TimeInterval FallbackCpuResolutionValue = TimeInterval.FromNanoseconds(0.2d);
29+
30+
public ExpectedBenchmarkResultsTests(ITestOutputHelper output) : base(output) { }
31+
32+
private static IEnumerable<Type> EmptyBenchmarkTypes() =>
33+
new[]
34+
{
35+
typeof(EmptyVoid),
36+
typeof(EmptyByte),
37+
typeof(EmptySByte),
38+
typeof(EmptyShort),
39+
typeof(EmptyUShort),
40+
typeof(EmptyChar),
41+
typeof(EmptyInt32),
42+
typeof(EmptyUInt32),
43+
typeof(EmptyInt64),
44+
typeof(EmptyUInt64),
45+
typeof(EmptyIntPtr),
46+
typeof(EmptyUIntPtr),
47+
typeof(EmptyVoidPointer),
48+
typeof(EmptyClass)
49+
};
50+
51+
public static IEnumerable<object[]> InProcessData()
52+
{
53+
foreach (var type in EmptyBenchmarkTypes())
54+
{
55+
yield return new object[] { type };
56+
}
57+
}
58+
59+
public static IEnumerable<object[]> CoreData()
60+
{
61+
foreach (var type in EmptyBenchmarkTypes())
62+
{
63+
yield return new object[] { type, RuntimeMoniker.Net70 };
64+
yield return new object[] { type, RuntimeMoniker.Mono70 };
65+
}
66+
}
67+
68+
public static IEnumerable<object[]> FrameworkData()
69+
{
70+
foreach (var type in EmptyBenchmarkTypes())
71+
{
72+
yield return new object[] { type, RuntimeMoniker.Net462 };
73+
yield return new object[] { type, RuntimeMoniker.Mono };
74+
}
75+
}
76+
77+
[Theory]
78+
[MemberData(nameof(InProcessData))]
79+
public void EmptyBenchmarksReportZeroTimeAndAllocated_InProcess(Type benchmarkType)
80+
{
81+
AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
82+
.AddJob(Job.Default
83+
.WithToolchain(InProcessEmitToolchain.Instance)
84+
));
85+
}
86+
87+
[TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")]
88+
[MemberData(nameof(CoreData))]
89+
public void EmptyBenchmarksReportZeroTimeAndAllocated_Core(Type benchmarkType, RuntimeMoniker runtimeMoniker)
90+
{
91+
AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
92+
.AddJob(Job.Default
93+
.WithRuntime(runtimeMoniker.GetRuntime())
94+
));
95+
}
96+
97+
[TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")]
98+
[MemberData(nameof(FrameworkData))]
99+
public void EmptyBenchmarksReportZeroTimeAndAllocated_Framework(Type benchmarkType, RuntimeMoniker runtimeMoniker)
100+
{
101+
AssertZeroResults(benchmarkType, ManualConfig.CreateEmpty()
102+
.AddJob(Job.Default
103+
.WithRuntime(runtimeMoniker.GetRuntime())
104+
));
105+
}
106+
107+
private void AssertZeroResults(Type benchmarkType, IConfig config)
108+
{
109+
var summary = CanExecute(benchmarkType, config
110+
.WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond))
111+
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
112+
);
113+
114+
var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue;
115+
var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds);
116+
117+
foreach (var report in summary.Reports)
118+
{
119+
var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers();
120+
var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers();
121+
122+
bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold);
123+
Assert.True(isZero, $"Actual time was not 0.");
124+
125+
isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold);
126+
Assert.True(isZero, "Overhead took more time than workload.");
127+
128+
Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");
129+
}
130+
}
131+
132+
[Fact]
133+
public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_InProcess()
134+
{
135+
AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty()
136+
.AddJob(Job.Default
137+
.WithToolchain(InProcessEmitToolchain.Instance)
138+
));
139+
}
140+
141+
[TheoryNetCoreOnly("To not repeat tests in both full Framework and Core")]
142+
[InlineData(RuntimeMoniker.Net70)]
143+
[InlineData(RuntimeMoniker.Mono70)]
144+
public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Core(RuntimeMoniker runtimeMoniker)
145+
{
146+
AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty()
147+
.AddJob(Job.Default
148+
.WithRuntime(runtimeMoniker.GetRuntime())
149+
));
150+
}
151+
152+
[TheoryFullFrameworkOnly("Can only run full Framework and Mono tests from Framework host")]
153+
[InlineData(RuntimeMoniker.Net462)]
154+
[InlineData(RuntimeMoniker.Mono)]
155+
public void DifferentSizedStructsBenchmarksReportsNonZeroTimeAndZeroAllocated_Framework(RuntimeMoniker runtimeMoniker)
156+
{
157+
AssertDifferentSizedStructsResults(ManualConfig.CreateEmpty()
158+
.AddJob(Job.Default
159+
.WithRuntime(runtimeMoniker.GetRuntime())
160+
));
161+
}
162+
163+
private void AssertDifferentSizedStructsResults(IConfig config)
164+
{
165+
var summary = CanExecute<DifferentSizedStructs>(config
166+
.WithSummaryStyle(SummaryStyle.Default.WithTimeUnit(TimeUnit.Nanosecond))
167+
.AddDiagnoser(new MemoryDiagnoser(new MemoryDiagnoserConfig(false)))
168+
);
169+
170+
var cpuResolution = RuntimeInformation.GetCpuInfo().MaxFrequency?.ToResolution() ?? FallbackCpuResolutionValue;
171+
var threshold = Threshold.Create(ThresholdUnit.Nanoseconds, cpuResolution.Nanoseconds);
172+
173+
foreach (var report in summary.Reports)
174+
{
175+
var workloadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).GetStatistics().WithoutOutliers();
176+
var overheadMeasurements = report.AllMeasurements.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual)).GetStatistics().WithoutOutliers();
177+
178+
bool isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workloadMeasurements, overheadMeasurements, threshold);
179+
Assert.False(isZero, $"Actual time was 0.");
180+
181+
isZero = ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(overheadMeasurements, workloadMeasurements, threshold);
182+
Assert.True(isZero, "Overhead took more time than workload.");
183+
184+
Assert.True((report.GcStats.GetBytesAllocatedPerOperation(report.BenchmarkCase) ?? 0L) == 0L, "Memory allocations measured above 0.");
185+
}
186+
}
187+
}
188+
189+
public struct Struct16
190+
{
191+
public long l1, l2;
192+
}
193+
194+
public struct Struct32
195+
{
196+
public long l1, l2, l3, l4;
197+
}
198+
199+
public struct Struct64
200+
{
201+
public long l1, l2, l3, l4,
202+
l5, l6, l7, l8;
203+
}
204+
205+
public struct Struct128
206+
{
207+
public long l1, l2, l3, l4,
208+
l5, l6, l7, l8,
209+
l9, l10, l11, l12,
210+
l13, l14, l15, l16;
211+
}
212+
213+
public class DifferentSizedStructs
214+
{
215+
[Benchmark] public Struct16 Struct16() => default;
216+
[Benchmark] public Struct32 Struct32() => default;
217+
[Benchmark] public Struct64 Struct64() => default;
218+
[Benchmark] public Struct128 Struct128() => default;
219+
}
220+
}
221+
222+
public class EmptyVoid { [Benchmark] public void Void() { } }
223+
public class EmptyByte { [Benchmark] public byte Byte() => default; }
224+
public class EmptySByte { [Benchmark] public sbyte SByte() => default; }
225+
public class EmptyShort { [Benchmark] public short Short() => default; }
226+
public class EmptyUShort { [Benchmark] public ushort UShort() => default; }
227+
public class EmptyChar { [Benchmark] public char Char() => default; }
228+
public class EmptyInt32 { [Benchmark] public int Int32() => default; }
229+
public class EmptyUInt32 { [Benchmark] public uint UInt32() => default; }
230+
public class EmptyInt64 { [Benchmark] public long Int64() => default; }
231+
public class EmptyUInt64 { [Benchmark] public ulong UInt64() => default; }
232+
public class EmptyIntPtr { [Benchmark] public IntPtr IntPtr() => default; }
233+
public class EmptyUIntPtr { [Benchmark] public UIntPtr UIntPtr() => default; }
234+
public class EmptyVoidPointer { [Benchmark] public unsafe void* VoidPointer() => default; }
235+
public class EmptyClass { [Benchmark] public object Class() => default; }

‎tests/BenchmarkDotNet.IntegrationTests/BenchmarkTestExecutor.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public Reports.Summary CanExecute<TBenchmark>(IConfig config = null, bool fullVa
5050
/// <param name="config">Optional custom config to be used instead of the default</param>
5151
/// <param name="fullValidation">Optional: disable validation (default = true/enabled)</param>
5252
/// <returns>The summary from the benchmark run</returns>
53-
protected Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true)
53+
public Reports.Summary CanExecute(Type type, IConfig config = null, bool fullValidation = true)
5454
{
5555
// Add logging, so the Benchmark execution is in the TestRunner output (makes Debugging easier)
5656
if (config == null)

0 commit comments

Comments
 (0)