From f3b3a9d0e5bdf190b638298a1f3072ac22b64fd1 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Wed, 12 Jan 2022 17:53:52 +0300 Subject: [PATCH 01/12] Move literals into a class; In future it will be used for cli code completion; StatisticalTest/HardwareCounters/CiLower are missing --- .../Columns/BaselineAllocationRatioColumn.cs | 2 +- src/BenchmarkDotNet/Columns/BaselineColumn.cs | 2 +- .../Columns/BaselineRatioColumn.cs | 4 +- .../Columns/CategoriesColumn.cs | 2 +- src/BenchmarkDotNet/Columns/Column.cs | 118 ++++++++++++++++++ .../Columns/JobCharacteristicColumn.cs | 2 +- .../Columns/LogicalGroupColumn.cs | 2 +- src/BenchmarkDotNet/Columns/RankColumn.cs | 2 +- .../Columns/StatisticColumn.cs | 50 ++++---- .../Columns/TargetMethodColumn.cs | 6 +- .../AllocatedMemoryMetricDescriptor.cs | 2 +- .../AllocatedNativeMemoryDescriptor.cs | 4 +- .../Diagnosers/MemoryDiagnoser.cs | 10 +- .../Diagnosers/ThreadingDiagnoser.cs | 4 +- .../Disassemblers/DisassemblyDiagnoser.cs | 2 +- 15 files changed, 165 insertions(+), 47 deletions(-) create mode 100644 src/BenchmarkDotNet/Columns/Column.cs diff --git a/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs b/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs index 78f89adc90..677f6acde1 100644 --- a/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs +++ b/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs @@ -12,7 +12,7 @@ namespace BenchmarkDotNet.Columns public class BaselineAllocationRatioColumn : BaselineCustomColumn { public override string Id => nameof(BaselineAllocationRatioColumn); - public override string ColumnName => "Alloc Ratio"; + public override string ColumnName => Column.AllocRatio; public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, IReadOnlyDictionary baselineMetrics, Statistics current, IReadOnlyDictionary currentMetrics, bool isBaseline) diff --git a/src/BenchmarkDotNet/Columns/BaselineColumn.cs b/src/BenchmarkDotNet/Columns/BaselineColumn.cs index c10a86e867..2154fb604b 100644 --- a/src/BenchmarkDotNet/Columns/BaselineColumn.cs +++ b/src/BenchmarkDotNet/Columns/BaselineColumn.cs @@ -9,7 +9,7 @@ public class BaselineColumn : IColumn [PublicAPI] public static readonly IColumn Default = new BaselineColumn(); public string Id => nameof(BaselineColumn); - public string ColumnName => "Baseline"; + public string ColumnName => Column.Baseline; public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => summary.IsBaseline(benchmarkCase) ? "Yes" : "No"; public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase); diff --git a/src/BenchmarkDotNet/Columns/BaselineRatioColumn.cs b/src/BenchmarkDotNet/Columns/BaselineRatioColumn.cs index d44738785d..4808a42821 100644 --- a/src/BenchmarkDotNet/Columns/BaselineRatioColumn.cs +++ b/src/BenchmarkDotNet/Columns/BaselineRatioColumn.cs @@ -35,9 +35,9 @@ public override string ColumnName switch (Metric) { case RatioMetric.Mean: - return "Ratio"; + return Column.Ratio; case RatioMetric.StdDev: - return "RatioSD"; + return Column.RatioSD; default: throw new NotSupportedException(); } diff --git a/src/BenchmarkDotNet/Columns/CategoriesColumn.cs b/src/BenchmarkDotNet/Columns/CategoriesColumn.cs index f47988278d..1b302434e0 100644 --- a/src/BenchmarkDotNet/Columns/CategoriesColumn.cs +++ b/src/BenchmarkDotNet/Columns/CategoriesColumn.cs @@ -10,7 +10,7 @@ public class CategoriesColumn : IColumn public static readonly IColumn Default = new CategoriesColumn(); public string Id => nameof(CategoriesColumn); - public string ColumnName => "Categories"; + public string ColumnName => Column.Categories; public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => string.Join(",", benchmarkCase.Descriptor.Categories); public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase); public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; diff --git a/src/BenchmarkDotNet/Columns/Column.cs b/src/BenchmarkDotNet/Columns/Column.cs new file mode 100644 index 0000000000..e9151881bd --- /dev/null +++ b/src/BenchmarkDotNet/Columns/Column.cs @@ -0,0 +1,118 @@ +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Columns +{ + // ReSharper disable once InconsistentNaming + [PublicAPI] + public static class Column + { + public const string Namespace = "Namespace"; + public const string Type = "Type"; + public const string Method = "Method"; + + public const string Job = "Job"; + + public const string Mean = "Mean"; + public const string StdErr = "StdErr"; + public const string StdDev = "StdDev"; + public const string Error = "Error"; + public const string OperationPerSecond = "Op/s"; + public const string Min = "Min"; + public const string Q1 = "Q1"; + public const string Median = "Median"; + public const string Q3 = "Q3"; + public const string Max = "Max"; + public const string Skewness = "Skewness"; + public const string Kurtosis = "Kurtosis"; + public const string MValue = "MValue"; + public const string Iterations = "Iterations"; + + public const string P0 = "P0"; + public const string P25 = "P25"; + public const string P50 = "P50"; + public const string P67 = "P67"; + public const string P80 = "P80"; + public const string P85 = "P85"; + public const string P90 = "P90"; + public const string P95 = "P95"; + public const string P100 = "P100"; + + public const string Categories = "Categories"; + public const string LogicalGroup = "LogicalGroup"; + public const string Rank = "Rank"; + + public const string Ratio = "Ratio"; + public const string RatioSD = "RatioSD"; + public const string AllocRatio = "Alloc Ratio"; + + public const string Allocated = "Allocated"; + public const string Gen0 = "Gen0"; + public const string Gen1 = "Gen1"; + public const string Gen2 = "Gen2"; + + public const string AllocatedNativeMemory = "Allocated native memory"; + public const string NativeMemoryLeak = "Native memory leak"; + public const string CompletedWorkItems = "Completed Work Items"; + public const string LockContentions = "Lock Contentions"; + public const string CodeSize = "Code Size"; + + //Characteristics: + public const string Id = "Id"; + + public const string MaxRelativeError = "MaxRelativeError"; + public const string MaxAbsoluteError = "MaxAbsoluteError"; + public const string MinIterationTime = "MinIterationTime"; + public const string MinInvokeCount = "MinInvokeCount"; + public const string EvaluateOverhead = "EvaluateOverhead"; + public const string OutlierMode = "OutlierMode"; + public const string AnalyzeLaunchVariance = "AnalyzeLaunchVariance"; + + public const string Platform = "Platform"; + public const string Jit = "Jit"; + public const string Runtime = "Runtime"; + public const string Affinity = "Affinity"; + public const string Gc = "Gc"; + public const string EnvironmentVariables = "EnvironmentVariables"; + public const string PowerPlanMode = "PowerPlanMode"; + + public const string Server = "Server"; + public const string Concurrent = "Concurrent"; + public const string CpuGroups = "CpuGroups"; + public const string Force = "Force"; + public const string AllowVeryLargeObjects = "AllowVeryLargeObjects"; + public const string RetainVm = "RetainVm"; + public const string NoAffinitize = "NoAffinitize"; + public const string HeapAffinitizeMask = "HeapAffinitizeMask"; + public const string HeapCount = "HeapCount"; + + public const string Toolchain = "Toolchain"; + public const string Clock = "Clock"; + public const string EngineFactory = "EngineFactory"; + public const string BuildConfiguration = "BuildConfiguration"; + public const string Arguments = "Arguments"; + public const string NuGetReferences = "NuGetReferences"; + + public const string Environment = "Environment"; + public const string Run = "Run"; + public const string Infrastructure = "Infrastructure"; + public const string Accuracy = "Accuracy"; + public const string Meta = "Meta"; + + public const string Baseline = "Baseline"; + public const string IsMutator = "IsMutator"; + public const string IsDefault = "IsDefault"; + + public const string RunStrategy = "RunStrategy"; + public const string LaunchCount = "LaunchCount"; + public const string InvocationCount = "InvocationCount"; + public const string UnrollFactor = "UnrollFactor"; + public const string IterationCount = "IterationCount"; + public const string MinIterationCount = "MinIterationCount"; + public const string MaxIterationCount = "MaxIterationCount"; + public const string IterationTime = "IterationTime"; + public const string WarmupCount = "WarmupCount"; + public const string MinWarmupIterationCount = "MinWarmupIterationCount"; + public const string MaxWarmupIterationCount = "MaxWarmupIterationCount"; + public const string MemoryRandomization = "MemoryRandomization"; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs b/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs index 74f7b2b8ba..75952b23bf 100644 --- a/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs +++ b/src/BenchmarkDotNet/Columns/JobCharacteristicColumn.cs @@ -23,7 +23,7 @@ private JobCharacteristicColumn(Characteristic characteristic) // The 'Id' characteristic is a special case: // here we just print 'Job' if (characteristic.Id == "Id") - ColumnName = "Job"; + ColumnName = Column.Job; } public string Id { get; } diff --git a/src/BenchmarkDotNet/Columns/LogicalGroupColumn.cs b/src/BenchmarkDotNet/Columns/LogicalGroupColumn.cs index a50cdd3ecd..3a0630e619 100644 --- a/src/BenchmarkDotNet/Columns/LogicalGroupColumn.cs +++ b/src/BenchmarkDotNet/Columns/LogicalGroupColumn.cs @@ -9,7 +9,7 @@ public class LogicalGroupColumn : IColumn [PublicAPI] public static readonly IColumn Default = new LogicalGroupColumn(); public string Id => nameof(LogicalGroupColumn); - public string ColumnName => "LogicalGroup"; + public string ColumnName => Column.LogicalGroup; public string GetValue(Summary summary, BenchmarkCase benchmarkCase) => summary.GetLogicalGroupKey(benchmarkCase); public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase); diff --git a/src/BenchmarkDotNet/Columns/RankColumn.cs b/src/BenchmarkDotNet/Columns/RankColumn.cs index 29879eaa84..6a9c9872c2 100644 --- a/src/BenchmarkDotNet/Columns/RankColumn.cs +++ b/src/BenchmarkDotNet/Columns/RankColumn.cs @@ -18,7 +18,7 @@ public class RankColumn : IColumn [PublicAPI] public static readonly IColumn Stars = new RankColumn(NumeralSystem.Stars); public string Id => nameof(RankColumn) + "." + numeralSystem; - public string ColumnName => "Rank"; + public string ColumnName => Column.Rank; public string GetValue(Summary summary, BenchmarkCase benchmarkCase) { diff --git a/src/BenchmarkDotNet/Columns/StatisticColumn.cs b/src/BenchmarkDotNet/Columns/StatisticColumn.cs index f3d37af8c6..31e72f2446 100644 --- a/src/BenchmarkDotNet/Columns/StatisticColumn.cs +++ b/src/BenchmarkDotNet/Columns/StatisticColumn.cs @@ -28,59 +28,59 @@ private enum Priority Additional } - public static readonly IStatisticColumn Mean = new StatisticColumn("Mean", "Arithmetic mean of all measurements", + public static readonly IStatisticColumn Mean = new StatisticColumn(Column.Mean, "Arithmetic mean of all measurements", s => s.Mean, Priority.Main); - public static readonly IColumn StdErr = new StatisticColumn("StdErr", "Standard error of all measurements", + public static readonly IColumn StdErr = new StatisticColumn(Column.StdErr, "Standard error of all measurements", s => s.StandardError, Priority.Main, parentColumn: Mean); - public static readonly IColumn StdDev = new StatisticColumn("StdDev", "Standard deviation of all measurements", + public static readonly IColumn StdDev = new StatisticColumn(Column.StdDev, "Standard deviation of all measurements", s => s.StandardDeviation, Priority.Main, parentColumn: Mean); - public static readonly IColumn Error = new StatisticColumn("Error", "Half of 99.9% confidence interval", + public static readonly IColumn Error = new StatisticColumn(Column.Error, "Half of 99.9% confidence interval", s => new ConfidenceInterval(s.Mean, s.StandardError, s.N, ConfidenceLevel.L999).Margin, Priority.Main, parentColumn: Mean); - public static readonly IColumn OperationsPerSecond = new StatisticColumn("Op/s", "Operation per second", + public static readonly IColumn OperationsPerSecond = new StatisticColumn(Column.OperationPerSecond, "Operation per second", s => 1.0 * 1000 * 1000 * 1000 / s.Mean, Priority.Additional, UnitType.Dimensionless); - public static readonly IColumn Min = new StatisticColumn("Min", "Minimum", + public static readonly IColumn Min = new StatisticColumn(Column.Min, "Minimum", s => s.Min, Priority.Quartile); - public static readonly IColumn Q1 = new StatisticColumn("Q1", "Quartile 1 (25th percentile)", + public static readonly IColumn Q1 = new StatisticColumn(Column.Q1, "Quartile 1 (25th percentile)", s => s.Q1, Priority.Quartile); - public static readonly IColumn Median = new StatisticColumn("Median", "Value separating the higher half of all measurements (50th percentile)", + public static readonly IColumn Median = new StatisticColumn(Column.Median, "Value separating the higher half of all measurements (50th percentile)", s => s.Median, Priority.Quartile); - public static readonly IColumn Q3 = new StatisticColumn("Q3", "Quartile 3 (75th percentile)", + public static readonly IColumn Q3 = new StatisticColumn(Column.Q3, "Quartile 3 (75th percentile)", s => s.Q3, Priority.Quartile); - public static readonly IColumn Max = new StatisticColumn("Max", "Maximum", s => s.Max, Priority.Quartile); + public static readonly IColumn Max = new StatisticColumn(Column.Max, "Maximum", s => s.Max, Priority.Quartile); - public static readonly IColumn Skewness = new StatisticColumn("Skewness", "Measure of the asymmetry (third standardized moment)", + public static readonly IColumn Skewness = new StatisticColumn(Column.Skewness, "Measure of the asymmetry (third standardized moment)", s => s.Skewness, Priority.Additional, UnitType.Dimensionless); - public static readonly IColumn Kurtosis = new StatisticColumn("Kurtosis", "Measure of the tailedness ( fourth standardized moment)", + public static readonly IColumn Kurtosis = new StatisticColumn(Column.Kurtosis, "Measure of the tailedness ( fourth standardized moment)", s => s.Kurtosis, Priority.Additional, UnitType.Dimensionless); /// /// See http://www.brendangregg.com/FrequencyTrails/modes.html /// - public static readonly IColumn MValue = new StatisticColumn("MValue", "Modal value, see http://www.brendangregg.com/FrequencyTrails/modes.html", + public static readonly IColumn MValue = new StatisticColumn(Column.MValue, "Modal value, see http://www.brendangregg.com/FrequencyTrails/modes.html", s => MValueCalculator.Calculate(s.OriginalValues), Priority.Additional, UnitType.Dimensionless); - public static readonly IColumn Iterations = new StatisticColumn("Iterations", "Number of target iterations", + public static readonly IColumn Iterations = new StatisticColumn(Column.Iterations, "Number of target iterations", s => s.N, Priority.Additional, UnitType.Dimensionless); - public static readonly IColumn P0 = CreatePercentileColumn(0, s => s.Percentiles.P0); - public static readonly IColumn P25 = CreatePercentileColumn(25, s => s.Percentiles.P25); - public static readonly IColumn P50 = CreatePercentileColumn(50, s => s.Percentiles.P50); - public static readonly IColumn P67 = CreatePercentileColumn(67, s => s.Percentiles.P67); - public static readonly IColumn P80 = CreatePercentileColumn(80, s => s.Percentiles.P80); - public static readonly IColumn P85 = CreatePercentileColumn(85, s => s.Percentiles.P85); - public static readonly IColumn P90 = CreatePercentileColumn(90, s => s.Percentiles.P90); - public static readonly IColumn P95 = CreatePercentileColumn(95, s => s.Percentiles.P95); - public static readonly IColumn P100 = CreatePercentileColumn(100, s => s.Percentiles.P100); + public static readonly IColumn P0 = CreatePercentileColumn(0, Column.P0, s => s.Percentiles.P0); + public static readonly IColumn P25 = CreatePercentileColumn(25, Column.P25, s => s.Percentiles.P25); + public static readonly IColumn P50 = CreatePercentileColumn(50, Column.P50, s => s.Percentiles.P50); + public static readonly IColumn P67 = CreatePercentileColumn(67, Column.P67, s => s.Percentiles.P67); + public static readonly IColumn P80 = CreatePercentileColumn(80, Column.P80, s => s.Percentiles.P80); + public static readonly IColumn P85 = CreatePercentileColumn(85, Column.P85, s => s.Percentiles.P85); + public static readonly IColumn P90 = CreatePercentileColumn(90, Column.P90, s => s.Percentiles.P90); + public static readonly IColumn P95 = CreatePercentileColumn(95, Column.P95, s => s.Percentiles.P95); + public static readonly IColumn P100 = CreatePercentileColumn(100, Column.P100, s => s.Percentiles.P100); [PublicAPI] public static IColumn CiLower(ConfidenceLevel level) => new StatisticColumn( @@ -165,7 +165,7 @@ private string Format(Summary summary, ImmutableConfig config, Statistics statis public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; - private static IColumn CreatePercentileColumn(int percentiles, Func calc) => new StatisticColumn( - "P" + percentiles, "Percentile " + percentiles, calc, Priority.Percentiles); + private static IColumn CreatePercentileColumn(int percentiles, string columnName, Func calc) => new StatisticColumn( + columnName, "Percentile " + percentiles, calc, Priority.Percentiles); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/TargetMethodColumn.cs b/src/BenchmarkDotNet/Columns/TargetMethodColumn.cs index 753970e09f..f6bcb3ef31 100644 --- a/src/BenchmarkDotNet/Columns/TargetMethodColumn.cs +++ b/src/BenchmarkDotNet/Columns/TargetMethodColumn.cs @@ -7,9 +7,9 @@ namespace BenchmarkDotNet.Columns { public class TargetMethodColumn : IColumn { - public static readonly IColumn Namespace = new TargetMethodColumn("Namespace", benchmark => benchmark.Descriptor.Type.Namespace); - public static readonly IColumn Type = new TargetMethodColumn("Type", benchmark => benchmark.Descriptor.Type.GetDisplayName()); - public static readonly IColumn Method = new TargetMethodColumn("Method", benchmark => benchmark.Descriptor.WorkloadMethodDisplayInfo, true); + public static readonly IColumn Namespace = new TargetMethodColumn(Column.Namespace, benchmark => benchmark.Descriptor.Type.Namespace); + public static readonly IColumn Type = new TargetMethodColumn(Column.Type, benchmark => benchmark.Descriptor.Type.GetDisplayName()); + public static readonly IColumn Method = new TargetMethodColumn(Column.Method, benchmark => benchmark.Descriptor.WorkloadMethodDisplayInfo, true); private readonly Func valueProvider; public string Id => nameof(TargetMethodColumn) + "." + ColumnName; diff --git a/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs b/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs index 0ff2a4bc55..0531d7e70b 100644 --- a/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs +++ b/src/BenchmarkDotNet/Diagnosers/AllocatedMemoryMetricDescriptor.cs @@ -9,7 +9,7 @@ internal class AllocatedMemoryMetricDescriptor : IMetricDescriptor internal static readonly IMetricDescriptor Instance = new AllocatedMemoryMetricDescriptor(); public string Id => "Allocated Memory"; - public string DisplayName => "Allocated"; + public string DisplayName => Column.Allocated; public string Legend => "Allocated memory per single operation (managed only, inclusive, 1KB = 1024B)"; public string NumberFormat => "0.##"; public UnitType UnitType => UnitType.Size; diff --git a/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs b/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs index 67e4e2d532..dec2d0a5bf 100644 --- a/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs +++ b/src/BenchmarkDotNet/Diagnosers/AllocatedNativeMemoryDescriptor.cs @@ -6,7 +6,7 @@ namespace BenchmarkDotNet.Diagnosers internal class AllocatedNativeMemoryDescriptor : IMetricDescriptor { public string Id => nameof(AllocatedNativeMemoryDescriptor); - public string DisplayName => $"Allocated native memory"; + public string DisplayName => Column.AllocatedNativeMemory; public string Legend => $"Allocated native memory per single operation"; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.Size; @@ -18,7 +18,7 @@ internal class AllocatedNativeMemoryDescriptor : IMetricDescriptor internal class NativeMemoryLeakDescriptor : IMetricDescriptor { public string Id => nameof(NativeMemoryLeakDescriptor); - public string DisplayName => $"Native memory leak"; + public string DisplayName => Column.NativeMemoryLeak; public string Legend => $"Native memory leak size in byte."; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.Size; diff --git a/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs index da1b0ec971..2bd39230a4 100644 --- a/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/MemoryDiagnoser.cs @@ -47,14 +47,14 @@ public IEnumerable ProcessResults(DiagnoserResults diagnoserResults) private class GarbageCollectionsMetricDescriptor : IMetricDescriptor { - internal static readonly IMetricDescriptor Gen0 = new GarbageCollectionsMetricDescriptor(0); - internal static readonly IMetricDescriptor Gen1 = new GarbageCollectionsMetricDescriptor(1); - internal static readonly IMetricDescriptor Gen2 = new GarbageCollectionsMetricDescriptor(2); + internal static readonly IMetricDescriptor Gen0 = new GarbageCollectionsMetricDescriptor(0, Column.Gen0); + internal static readonly IMetricDescriptor Gen1 = new GarbageCollectionsMetricDescriptor(1, Column.Gen1); + internal static readonly IMetricDescriptor Gen2 = new GarbageCollectionsMetricDescriptor(2, Column.Gen2); - private GarbageCollectionsMetricDescriptor(int generationId) + private GarbageCollectionsMetricDescriptor(int generationId, string columnName) { Id = $"Gen{generationId}Collects"; - DisplayName = $"Gen {generationId}"; + DisplayName = columnName; Legend = $"GC Generation {generationId} collects per 1000 operations"; PriorityInCategory = generationId; } diff --git a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs index 017ddde4b6..cf19cba1e1 100644 --- a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs @@ -55,7 +55,7 @@ private class CompletedWorkItemCountMetricDescriptor : IMetricDescriptor internal static readonly IMetricDescriptor Instance = new CompletedWorkItemCountMetricDescriptor(); public string Id => "CompletedWorkItemCount"; - public string DisplayName => "Completed Work Items"; + public string DisplayName => Column.CompletedWorkItems; public string Legend => "The number of work items that have been processed in ThreadPool (per single operation)"; public string NumberFormat => "#0.0000"; public UnitType UnitType => UnitType.Dimensionless; @@ -69,7 +69,7 @@ private class LockContentionCountMetricDescriptor : IMetricDescriptor internal static readonly IMetricDescriptor Instance = new LockContentionCountMetricDescriptor(); public string Id => "LockContentionCount"; - public string DisplayName => "Lock Contentions"; + public string DisplayName => Column.LockContentions; public string Legend => "The number of times there was contention upon trying to take a Monitor's lock (per single operation)"; public string NumberFormat => "#0.0000"; public UnitType UnitType => UnitType.Dimensionless; diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs index a11aece4ea..f235ac48f5 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs @@ -163,7 +163,7 @@ private class NativeCodeSizeMetricDescriptor : IMetricDescriptor internal static readonly IMetricDescriptor Instance = new NativeCodeSizeMetricDescriptor(); public string Id => "Native Code Size"; - public string DisplayName => "Code Size"; + public string DisplayName => Column.CodeSize; public string Legend => "Native code size of the disassembled method(s)"; public string NumberFormat => "N0"; public UnitType UnitType => UnitType.CodeSize; From dd10a4e4d5ec121955076c89895ec1739b37636f Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Mon, 10 Jan 2022 12:59:01 +0300 Subject: [PATCH 02/12] Add HideColumns --- .../Attributes/HideColumnsAttribute.cs | 27 +++++++++++++++++++ .../Columns/ColumnHidingByIdRule.cs | 14 ++++++++++ .../Columns/ColumnHidingByNameRule.cs | 14 ++++++++++ .../Columns/IColumnHidingRule.cs | 7 +++++ .../Configs/ConfigExtensions.cs | 6 +++++ src/BenchmarkDotNet/Configs/DebugConfig.cs | 1 + src/BenchmarkDotNet/Configs/DefaultConfig.cs | 2 ++ src/BenchmarkDotNet/Configs/IConfig.cs | 1 + .../Configs/ImmutableConfig.cs | 4 +++ .../Configs/ImmutableConfigBuilder.cs | 2 ++ src/BenchmarkDotNet/Configs/ManualConfig.cs | 10 +++++++ src/BenchmarkDotNet/Reports/Summary.cs | 13 ++++++--- src/BenchmarkDotNet/Reports/SummaryTable.cs | 15 ++++++++--- .../Reports/SummaryTableExtensions.cs | 2 +- .../Running/BenchmarkRunnerClean.cs | 4 ++- 15 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 src/BenchmarkDotNet/Attributes/HideColumnsAttribute.cs create mode 100644 src/BenchmarkDotNet/Columns/ColumnHidingByIdRule.cs create mode 100644 src/BenchmarkDotNet/Columns/ColumnHidingByNameRule.cs create mode 100644 src/BenchmarkDotNet/Columns/IColumnHidingRule.cs diff --git a/src/BenchmarkDotNet/Attributes/HideColumnsAttribute.cs b/src/BenchmarkDotNet/Attributes/HideColumnsAttribute.cs new file mode 100644 index 0000000000..f66350e962 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/HideColumnsAttribute.cs @@ -0,0 +1,27 @@ +using System; +using BenchmarkDotNet.Configs; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Attributes +{ + [PublicAPI] + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + public class HideColumnsAttribute : Attribute, IConfigSource + { + public string[] Names { get; } + + public IConfig Config { get; } + + // CLS-Compliant Code requires a constructor without an array in the argument list + protected HideColumnsAttribute() + { + Config = ManualConfig.CreateEmpty(); + } + + public HideColumnsAttribute(params string[] names) + { + Names = names; + Config = ManualConfig.CreateEmpty().HideColumns(names); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/ColumnHidingByIdRule.cs b/src/BenchmarkDotNet/Columns/ColumnHidingByIdRule.cs new file mode 100644 index 0000000000..2e54665bf3 --- /dev/null +++ b/src/BenchmarkDotNet/Columns/ColumnHidingByIdRule.cs @@ -0,0 +1,14 @@ +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Columns +{ + [PublicAPI] + internal class ColumnHidingByIdRule: IColumnHidingRule + { + public string Id { get; } + + public ColumnHidingByIdRule(IColumn column) => Id = column.Id; + + public bool NeedToHide(IColumn column) => column.Id == Id; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/ColumnHidingByNameRule.cs b/src/BenchmarkDotNet/Columns/ColumnHidingByNameRule.cs new file mode 100644 index 0000000000..c309ab9f94 --- /dev/null +++ b/src/BenchmarkDotNet/Columns/ColumnHidingByNameRule.cs @@ -0,0 +1,14 @@ +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Columns +{ + [PublicAPI] + internal class ColumnHidingByNameRule: IColumnHidingRule + { + public string Name { get; } + + public ColumnHidingByNameRule(string name) => Name = name; + + public bool NeedToHide(IColumn column) => column.ColumnName == Name; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/IColumnHidingRule.cs b/src/BenchmarkDotNet/Columns/IColumnHidingRule.cs new file mode 100644 index 0000000000..340d3883ed --- /dev/null +++ b/src/BenchmarkDotNet/Columns/IColumnHidingRule.cs @@ -0,0 +1,7 @@ +namespace BenchmarkDotNet.Columns +{ + public interface IColumnHidingRule + { + bool NeedToHide(IColumn column); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs index 9c903566f0..b5702c3d9c 100644 --- a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs @@ -111,6 +111,12 @@ public static class ConfigExtensions [EditorBrowsable(EditorBrowsableState.Never)] public static IConfig With(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.AddLogicalGroupRules(rules); [PublicAPI] public static ManualConfig AddLogicalGroupRules(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.With(c => c.AddLogicalGroupRules(rules)); + [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params IColumn[] columns) => config.With(c => c.HideColumns(columns.Select(n => (IColumnHidingRule)new ColumnHidingByIdRule(n)).ToArray())); + + [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params string[] columnNames) => config.With(c => c.HideColumns(columnNames.Select(n => (IColumnHidingRule)new ColumnHidingByNameRule(n)).ToArray())); + + [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params IColumnHidingRule[] rules) => config.With(c => c.HideColumns(rules)); + public static ImmutableConfig CreateImmutableConfig(this IConfig config) => ImmutableConfigBuilder.Create(config); internal static ILogger GetNonNullCompositeLogger(this IConfig config) diff --git a/src/BenchmarkDotNet/Configs/DebugConfig.cs b/src/BenchmarkDotNet/Configs/DebugConfig.cs index 19295fed58..19868fb585 100644 --- a/src/BenchmarkDotNet/Configs/DebugConfig.cs +++ b/src/BenchmarkDotNet/Configs/DebugConfig.cs @@ -62,6 +62,7 @@ public abstract class DebugConfig : IConfig public IEnumerable GetAnalysers() => Array.Empty(); public IEnumerable GetHardwareCounters() => Array.Empty(); public IEnumerable GetFilters() => Array.Empty(); + public IEnumerable GetColumnHidingRules() => Array.Empty(); public IOrderer Orderer => DefaultOrderer.Instance; public SummaryStyle SummaryStyle => SummaryStyle.Default; diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index eb3fb6e80a..fa4baee771 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -98,5 +98,7 @@ public string ArtifactsPath public IEnumerable GetHardwareCounters() => Array.Empty(); public IEnumerable GetFilters() => Array.Empty(); + + public IEnumerable GetColumnHidingRules() => Array.Empty(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/IConfig.cs b/src/BenchmarkDotNet/Configs/IConfig.cs index bcfd56a60c..2075b632c6 100644 --- a/src/BenchmarkDotNet/Configs/IConfig.cs +++ b/src/BenchmarkDotNet/Configs/IConfig.cs @@ -26,6 +26,7 @@ public interface IConfig IEnumerable GetHardwareCounters(); IEnumerable GetFilters(); IEnumerable GetLogicalGroupRules(); + IEnumerable GetColumnHidingRules(); [CanBeNull] IOrderer Orderer { get; } diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs index b1e74668bf..1d74164153 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfig.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfig.cs @@ -31,6 +31,7 @@ public sealed class ImmutableConfig : IConfig private readonly ImmutableHashSet hardwareCounters; private readonly ImmutableHashSet filters; private readonly ImmutableArray rules; + private readonly ImmutableArray columnHidingRules; internal ImmutableConfig( ImmutableArray uniqueColumnProviders, @@ -42,6 +43,7 @@ internal ImmutableConfig( ImmutableHashSet uniqueValidators, ImmutableHashSet uniqueFilters, ImmutableArray uniqueRules, + ImmutableArray uniqueColumnHidingRules, ImmutableHashSet uniqueRunnableJobs, ConfigUnionRule unionRule, string artifactsPath, @@ -59,6 +61,7 @@ internal ImmutableConfig( validators = uniqueValidators; filters = uniqueFilters; rules = uniqueRules; + columnHidingRules = uniqueColumnHidingRules; jobs = uniqueRunnableJobs; UnionRule = unionRule; ArtifactsPath = artifactsPath; @@ -85,6 +88,7 @@ internal ImmutableConfig( public IEnumerable GetHardwareCounters() => hardwareCounters; public IEnumerable GetFilters() => filters; public IEnumerable GetLogicalGroupRules() => rules; + public IEnumerable GetColumnHidingRules() => columnHidingRules; public ILogger GetCompositeLogger() => new CompositeLogger(loggers); public IExporter GetCompositeExporter() => new CompositeExporter(exporters); diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index a04f26f804..5ec2f2b3d0 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -46,6 +46,7 @@ public static ImmutableConfig Create(IConfig source) var uniqueFilters = source.GetFilters().ToImmutableHashSet(); var uniqueRules = source.GetLogicalGroupRules().ToImmutableArray(); + var uniqueHidingRules = source.GetColumnHidingRules().ToImmutableArray(); var uniqueRunnableJobs = GetRunnableJobs(source.GetJobs()).ToImmutableHashSet(); @@ -59,6 +60,7 @@ public static ImmutableConfig Create(IConfig source) uniqueValidators, uniqueFilters, uniqueRules, + uniqueHidingRules, uniqueRunnableJobs, source.UnionRule, source.ArtifactsPath ?? DefaultConfig.Instance.ArtifactsPath, diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index 7a46d6450e..f2a16b2944 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -30,6 +30,7 @@ public class ManualConfig : IConfig private readonly HashSet hardwareCounters = new HashSet(); private readonly List filters = new List(); private readonly List logicalGroupRules = new List(); + private readonly List columnHidingRules = new List(); public IEnumerable GetColumnProviders() => columnProviders; public IEnumerable GetExporters() => exporters; @@ -41,6 +42,7 @@ public class ManualConfig : IConfig public IEnumerable GetHardwareCounters() => hardwareCounters; public IEnumerable GetFilters() => filters; public IEnumerable GetLogicalGroupRules() => logicalGroupRules; + public IEnumerable GetColumnHidingRules() => columnHidingRules; [PublicAPI] public ConfigOptions Options { get; set; } [PublicAPI] public ConfigUnionRule UnionRule { get; set; } = ConfigUnionRule.Union; @@ -200,6 +202,13 @@ public ManualConfig AddLogicalGroupRules(params BenchmarkLogicalGroupRule[] rule return this; } + [PublicAPI] + public ManualConfig HideColumns(params IColumnHidingRule[] rules) + { + columnHidingRules.AddRange(rules); + return this; + } + [PublicAPI] public void Add(IConfig config) { @@ -217,6 +226,7 @@ public void Add(IConfig config) CultureInfo = config.CultureInfo ?? CultureInfo; SummaryStyle = config.SummaryStyle ?? SummaryStyle; logicalGroupRules.AddRange(config.GetLogicalGroupRules()); + columnHidingRules.AddRange(config.GetColumnHidingRules()); Options |= config.Options; } diff --git a/src/BenchmarkDotNet/Reports/Summary.cs b/src/BenchmarkDotNet/Reports/Summary.cs index 87e96695eb..9138896e36 100644 --- a/src/BenchmarkDotNet/Reports/Summary.cs +++ b/src/BenchmarkDotNet/Reports/Summary.cs @@ -3,6 +3,7 @@ using System.Collections.Immutable; using System.Globalization; using System.Linq; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; @@ -26,6 +27,7 @@ public class Summary [PublicAPI] public SummaryTable Table { get; } [PublicAPI] public string AllRuntimes { get; } [PublicAPI] public ImmutableArray ValidationErrors { get; } + [PublicAPI] public ImmutableArray ColumnHidingRules { get; } [PublicAPI] public ImmutableArray BenchmarksCases { get; } [PublicAPI] public ImmutableArray Reports { get; } @@ -43,7 +45,8 @@ public Summary( string logFilePath, TimeSpan totalTime, CultureInfo cultureInfo, - ImmutableArray validationErrors) + ImmutableArray validationErrors, + ImmutableArray columnHidingRules) { Title = title; ResultsDirectoryPath = resultsDirectoryPath; @@ -51,6 +54,7 @@ public Summary( HostEnvironmentInfo = hostEnvironmentInfo; TotalTime = totalTime; ValidationErrors = validationErrors; + ColumnHidingRules = columnHidingRules; ReportMap = reports.ToImmutableDictionary(report => report.BenchmarkCase, report => report); @@ -76,10 +80,10 @@ public Summary( public int GetNumberOfExecutedBenchmarks() => Reports.Count(report => report.ExecuteResults.Any(result => result.FoundExecutable)); internal static Summary NothingToRun(string title, string resultsDirectoryPath, string logFilePath) - => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, ImmutableArray.Empty); + => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, ImmutableArray.Empty, ImmutableArray.Empty); internal static Summary ValidationFailed(string title, string resultsDirectoryPath, string logFilePath, ImmutableArray validationErrors) - => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors); + => new Summary(title, ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), resultsDirectoryPath, logFilePath, TimeSpan.Zero, DefaultCultureInfo.Instance, validationErrors, ImmutableArray.Empty); internal static Summary Join(List summaries, ClockSpan clockSpan) => new Summary( @@ -90,7 +94,8 @@ internal static Summary Join(List summaries, ClockSpan clockSpan) summaries.First().LogFilePath, clockSpan.GetTimeSpan(), summaries.First().GetCultureInfo(), - summaries.SelectMany(summary => summary.ValidationErrors).ToImmutableArray()); + summaries.SelectMany(summary => summary.ValidationErrors).ToImmutableArray(), + summaries.SelectMany(summary => summary.ColumnHidingRules).ToImmutableArray()); internal static string BuildAllRuntimes(HostEnvironmentInfo hostEnvironmentInfo, IEnumerable reports) { diff --git a/src/BenchmarkDotNet/Reports/SummaryTable.cs b/src/BenchmarkDotNet/Reports/SummaryTable.cs index c4bca02703..ee77ab877d 100644 --- a/src/BenchmarkDotNet/Reports/SummaryTable.cs +++ b/src/BenchmarkDotNet/Reports/SummaryTable.cs @@ -86,7 +86,14 @@ internal SummaryTable(Summary summary, SummaryStyle style = null) full.AddRange(FullContent); FullContentWithHeader = full.ToArray(); - Columns = Enumerable.Range(0, columns.Length).Select(i => new SummaryTableColumn(this, i, columns[i])).ToArray(); + Columns = new SummaryTableColumn[columns.Length]; + for (int i = 0; i < columns.Length; i++) + { + var column = columns[i]; + bool hide = summary.ColumnHidingRules.Any(rule => rule.NeedToHide(column)); + Columns[i] = new SummaryTableColumn(this, i, column, hide); + } + EffectiveSummaryStyle = style; } @@ -95,18 +102,20 @@ public class SummaryTableColumn [PublicAPI] public int Index { get; } public string Header { get; } public string[] Content { get; } + public bool IsHidden { get; } public bool NeedToShow { get; } public int Width { get; } public bool IsDefault { get; } public TextJustification Justify { get; } public IColumn OriginalColumn { get; } - public SummaryTableColumn(SummaryTable table, int index, IColumn column) + public SummaryTableColumn(SummaryTable table, int index, IColumn column, bool hide = false) { Index = index; Header = table.FullHeader[index]; Content = table.FullContent.Select(line => line[index]).ToArray(); - NeedToShow = column.AlwaysShow || Content.Distinct().Count() > 1; + IsHidden = hide; + NeedToShow = !hide && (column.AlwaysShow || Content.Distinct().Count() > 1); Width = Math.Max(Header.Length, Content.Any() ? Content.Max(line => line.Length) : 0) + 1; IsDefault = table.IsDefault[index]; OriginalColumn = column; diff --git a/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs b/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs index f8f8fdc3aa..cafb089ca1 100644 --- a/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs +++ b/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs @@ -15,7 +15,7 @@ public static class SummaryTableExtensions public static void PrintCommonColumns(this SummaryTable table, ILogger logger) { - var commonColumns = table.Columns.Where(c => !c.NeedToShow && !c.IsDefault).ToArray(); + var commonColumns = table.Columns.Where(c => (c.IsHidden && c.Content.Distinct().Count() == 1) || (!c.IsHidden && !c.NeedToShow && !c.IsDefault)).ToArray(); if (commonColumns.Any()) { int paramsOnLine = 0; diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 8bc61a3c9b..bd8e494ee5 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -5,6 +5,7 @@ using System.Linq; using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; @@ -209,7 +210,8 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, logFilePath, clockSpan.GetTimeSpan(), cultureInfo, - Validate(new[] {benchmarkRunInfo }, NullLogger.Instance)); // validate them once again, but don't print the output + Validate(new[] {benchmarkRunInfo }, NullLogger.Instance), // validate them once again, but don't print the output + benchmarkRunInfo.Config.GetColumnHidingRules().ToImmutableArray()); } private static void PrintSummary(ILogger logger, ImmutableConfig config, Summary summary) From 4e2568f501891ba7bc8a39dbb9b75d9ed83c16b5 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Mon, 10 Jan 2022 13:00:32 +0300 Subject: [PATCH 03/12] Fix tests --- .../ExporterIOTests.cs | 4 +++- tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs | 10 +++++++--- .../BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs | 5 +++-- .../Reports/RatioPrecisionTests.cs | 3 ++- tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs | 3 ++- tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs | 2 +- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs index c2ebe695fb..9b2bdbe338 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ExporterIOTests.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Loggers; @@ -128,7 +129,8 @@ private Summary GetMockSummary(string resultsDirectoryPath, IConfig config, para logFilePath: string.Empty, totalTime: System.TimeSpan.Zero, cultureInfo: TestCultureInfo.Instance, - validationErrors: ImmutableArray.Empty + validationErrors: ImmutableArray.Empty, + columnHidingRules: ImmutableArray.Empty ); } diff --git a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs index 448ed60804..c5b42ee06c 100644 --- a/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Mocks/MockFactory.cs @@ -3,6 +3,7 @@ using System.Collections.Immutable; using System.Linq; using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Reports; @@ -27,7 +28,8 @@ public static Summary CreateSummary(Type benchmarkType) string.Empty, TimeSpan.FromMinutes(1), TestCultureInfo.Instance, - ImmutableArray.Empty); + ImmutableArray.Empty, + ImmutableArray.Empty); } public static Summary CreateSummary(IConfig config) => new Summary( @@ -38,7 +40,8 @@ public static Summary CreateSummary(Type benchmarkType) string.Empty, TimeSpan.FromMinutes(1), config.CultureInfo, - ImmutableArray.Empty); + ImmutableArray.Empty, + ImmutableArray.Empty); public static Summary CreateSummary(IConfig config, bool hugeSd, Metric[] metrics) => CreateSummary(config, hugeSd, metrics); @@ -51,7 +54,8 @@ public static Summary CreateSummary(IConfig config, bool hugeSd, Metric[] metric string.Empty, TimeSpan.FromMinutes(1), TestCultureInfo.Instance, - ImmutableArray.Empty); + ImmutableArray.Empty, + ImmutableArray.Empty); private static ImmutableArray CreateReports(IConfig config) => CreateBenchmarks(config).Select(CreateSimpleReport).ToImmutableArray(); diff --git a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs index ed3b0aed98..43b588e019 100644 --- a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs +++ b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs @@ -3,6 +3,7 @@ using System.Collections.Immutable; using System.Globalization; using System.Linq; +using BenchmarkDotNet.Columns; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; @@ -18,9 +19,9 @@ namespace BenchmarkDotNet.Tests.Order public class DefaultOrdererTests { private static Summary CreateMockSummary() => new ("", ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), - "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty); + "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty, ImmutableArray.Empty); - private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new ( + private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new( new Descriptor(null, null, categories: new[] { category }), new Job(), new ParameterInstances(new[] diff --git a/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs b/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs index c22819ef91..9821cd2018 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/RatioPrecisionTests.cs @@ -72,7 +72,8 @@ private Summary CreateSummary(int[] values) string.Empty, TimeSpan.FromMinutes(1), TestCultureInfo.Instance, - ImmutableArray.Empty); + ImmutableArray.Empty, + ImmutableArray.Empty); MarkdownExporter.Default.ExportToLog(summary, logger); output.WriteLine(logger.GetLog()); return summary; diff --git a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs index 7fb0c16f5b..bbd2267d05 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/RatioStyleTests.cs @@ -93,7 +93,8 @@ private Summary CreateSummary(int[] values, RatioStyle ratioStyle, int noise) string.Empty, TimeSpan.FromMinutes(1), TestCultureInfo.Instance, - ImmutableArray.Empty); + ImmutableArray.Empty, + ImmutableArray.Empty); MarkdownExporter.Default.ExportToLog(summary, logger); output.WriteLine(logger.GetLog()); return summary; diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs index 8497f83bd4..c2fd8464c4 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs @@ -74,7 +74,7 @@ private static BenchmarkReport CreateSuccessReport(BenchmarkCase benchmark) private static Summary CreateSummary(IList reports) { HostEnvironmentInfo hostEnvironmentInfo = new HostEnvironmentInfoBuilder().Build(); - return new Summary("MockSummary", reports.ToImmutableArray(), hostEnvironmentInfo, string.Empty, string.Empty, TimeSpan.FromMinutes(1.0), TestCultureInfo.Instance, ImmutableArray.Empty); + return new Summary("MockSummary", reports.ToImmutableArray(), hostEnvironmentInfo, string.Empty, string.Empty, TimeSpan.FromMinutes(1.0), TestCultureInfo.Instance, ImmutableArray.Empty, ImmutableArray.Empty); } public class MockBenchmarkClass From 71aa462cd7d772b045698db844d3a14c1bee9064 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Tue, 11 Jan 2022 09:25:05 +0300 Subject: [PATCH 04/12] Do not start printing a table when no columns, details: When all columns are disabled, it was printing: | | | Replace it with the message, which is in contrast with errors --- src/BenchmarkDotNet/Exporters/MarkdownExporter.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs b/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs index d846732d02..8435bbe8d7 100644 --- a/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs +++ b/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs @@ -139,6 +139,12 @@ private void PrintTable(SummaryTable table, ILogger logger) return; } + if (table.Columns.All(c => !c.NeedToShow)) + { + logger.WriteLine("There are no columns to show "); + return; + } + table.PrintCommonColumns(logger); logger.WriteLine(); From 96dd73e230cb5ed2b363d07ebd7b583e90696fcd Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Tue, 11 Jan 2022 09:09:14 +0300 Subject: [PATCH 05/12] Show legends only for visible columns i.e. hide the time legend when Mean&Error are hidden --- src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index bd8e494ee5..fc4e9549ee 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -241,8 +241,11 @@ private static void PrintSummary(ILogger logger, ImmutableConfig config, Summary ConclusionHelper.Print(logger, config.GetCompositeAnalyser().Analyse(summary).Distinct().ToList()); // TODO: move to conclusions - var columnWithLegends = summary.Table.Columns.Select(c => c.OriginalColumn).Where(c => !string.IsNullOrEmpty(c.Legend)).ToList(); - var effectiveTimeUnit = summary.Table.EffectiveSummaryStyle.TimeUnit; + var columnWithLegends = summary.Table.Columns.Where(c => c.NeedToShow).Select(c => c.OriginalColumn).Where(c => !string.IsNullOrEmpty(c.Legend)).ToList(); + + bool needToShowTimeLegend = summary.Table.Columns.Where(c => c.NeedToShow).Select(c => c.OriginalColumn).Any(c => c.UnitType == UnitType.Time); + var effectiveTimeUnit = needToShowTimeLegend ? summary.Table.EffectiveSummaryStyle.TimeUnit : null; + if (columnWithLegends.Any() || effectiveTimeUnit != null) { logger.WriteLine(); From f66f78936f8242389c4c72218683096b78ca5e1a Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Tue, 11 Jan 2022 11:30:30 +0300 Subject: [PATCH 06/12] Replace AllocRatio ctor creation to the static field, like BaselineRatioColumn.RatioMean --- src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs | 4 ++++ src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs b/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs index 677f6acde1..a7e4052ee7 100644 --- a/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs +++ b/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs @@ -14,6 +14,10 @@ public class BaselineAllocationRatioColumn : BaselineCustomColumn public override string Id => nameof(BaselineAllocationRatioColumn); public override string ColumnName => Column.AllocRatio; + public static readonly IColumn RatioMean = new BaselineAllocationRatioColumn(); + + private BaselineAllocationRatioColumn() { } + public override string GetValue(Summary summary, BenchmarkCase benchmarkCase, Statistics baseline, IReadOnlyDictionary baselineMetrics, Statistics current, IReadOnlyDictionary currentMetrics, bool isBaseline) { diff --git a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs index b69264901b..0677f73c77 100644 --- a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs +++ b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs @@ -66,7 +66,7 @@ public IEnumerable GetColumns(Summary summary) if (HasMemoryDiagnoser(summary)) { - yield return new BaselineAllocationRatioColumn(); + yield return BaselineAllocationRatioColumn.RatioMean; } } } From 549cefc3e75fc1001cd8cbff6e81c08c1dc9374b Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Sat, 29 Jan 2022 21:32:07 +0300 Subject: [PATCH 07/12] Empty; merge conflict resolved From f2ca9f407e95b8d984f853d5537aa235842e4b3a Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Sat, 29 Jan 2022 20:27:50 +0300 Subject: [PATCH 08/12] Encapsulate tight logic in SummaryTableColumn --- src/BenchmarkDotNet/Reports/SummaryTable.cs | 13 ++++++++++--- .../Reports/SummaryTableExtensions.cs | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/BenchmarkDotNet/Reports/SummaryTable.cs b/src/BenchmarkDotNet/Reports/SummaryTable.cs index ee77ab877d..bb2c411b5e 100644 --- a/src/BenchmarkDotNet/Reports/SummaryTable.cs +++ b/src/BenchmarkDotNet/Reports/SummaryTable.cs @@ -102,25 +102,32 @@ public class SummaryTableColumn [PublicAPI] public int Index { get; } public string Header { get; } public string[] Content { get; } - public bool IsHidden { get; } public bool NeedToShow { get; } public int Width { get; } public bool IsDefault { get; } public TextJustification Justify { get; } public IColumn OriginalColumn { get; } + internal bool IsCommonColumn { get; } + internal bool WasHidden { get; } + public SummaryTableColumn(SummaryTable table, int index, IColumn column, bool hide = false) { Index = index; Header = table.FullHeader[index]; Content = table.FullContent.Select(line => line[index]).ToArray(); - IsHidden = hide; - NeedToShow = !hide && (column.AlwaysShow || Content.Distinct().Count() > 1); Width = Math.Max(Header.Length, Content.Any() ? Content.Max(line => line.Length) : 0) + 1; IsDefault = table.IsDefault[index]; OriginalColumn = column; Justify = column.IsNumeric ? TextJustification.Right : TextJustification.Left; + + bool needToShow = column.AlwaysShow || Content.Distinct().Count() > 1; + NeedToShow = !hide && needToShow; + WasHidden = hide && needToShow; + + bool isCommonColumn = !NeedToShow && !IsDefault; + IsCommonColumn = (!hide && isCommonColumn) || (hide && Content.Distinct().Count() == 1); } public override string ToString() => Header; diff --git a/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs b/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs index cafb089ca1..3829602443 100644 --- a/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs +++ b/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs @@ -15,7 +15,7 @@ public static class SummaryTableExtensions public static void PrintCommonColumns(this SummaryTable table, ILogger logger) { - var commonColumns = table.Columns.Where(c => (c.IsHidden && c.Content.Distinct().Count() == 1) || (!c.IsHidden && !c.NeedToShow && !c.IsDefault)).ToArray(); + var commonColumns = table.Columns.Where(c => c.IsCommonColumn).ToArray(); if (commonColumns.Any()) { int paramsOnLine = 0; From 728365edd8d4124b36f0649e69b0ceaa7ac246e0 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Sat, 29 Jan 2022 20:24:02 +0300 Subject: [PATCH 09/12] Add CLI support --- .../Analysers/HideColumnsAnalyser.cs | 27 ++++++++++++ .../Configs/ConfigExtensions.cs | 6 +-- src/BenchmarkDotNet/Configs/DefaultConfig.cs | 1 + src/BenchmarkDotNet/Configs/ManualConfig.cs | 14 +++++++ .../ConsoleArguments/CommandLineOptions.cs | 6 ++- .../ConsoleArguments/ConfigParser.cs | 42 ++++++++++--------- 6 files changed, 71 insertions(+), 25 deletions(-) create mode 100644 src/BenchmarkDotNet/Analysers/HideColumnsAnalyser.cs diff --git a/src/BenchmarkDotNet/Analysers/HideColumnsAnalyser.cs b/src/BenchmarkDotNet/Analysers/HideColumnsAnalyser.cs new file mode 100644 index 0000000000..9db11cc7a7 --- /dev/null +++ b/src/BenchmarkDotNet/Analysers/HideColumnsAnalyser.cs @@ -0,0 +1,27 @@ +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Reports; + +namespace BenchmarkDotNet.Analysers +{ + public class HideColumnsAnalyser : AnalyserBase + { + public static readonly IAnalyser Default = new HideColumnsAnalyser(); + + public override string Id => nameof(HideColumnsAnalyser); + + protected override IEnumerable AnalyseSummary(Summary summary) + { + var hiddenColumns = summary.GetTable(summary.Style).Columns.Where(c => c.WasHidden).ToArray(); + + if (hiddenColumns.IsEmpty()) + yield break; + + var columnNames = string.Join(", ", hiddenColumns.Select(c => c.OriginalColumn.ColumnName)); + + var message = $"Hidden columns: {columnNames}"; + yield return Conclusion.CreateHint(Id, message); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs index b5702c3d9c..fa0920c7f9 100644 --- a/src/BenchmarkDotNet/Configs/ConfigExtensions.cs +++ b/src/BenchmarkDotNet/Configs/ConfigExtensions.cs @@ -111,10 +111,8 @@ public static class ConfigExtensions [EditorBrowsable(EditorBrowsableState.Never)] public static IConfig With(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.AddLogicalGroupRules(rules); [PublicAPI] public static ManualConfig AddLogicalGroupRules(this IConfig config, params BenchmarkLogicalGroupRule[] rules) => config.With(c => c.AddLogicalGroupRules(rules)); - [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params IColumn[] columns) => config.With(c => c.HideColumns(columns.Select(n => (IColumnHidingRule)new ColumnHidingByIdRule(n)).ToArray())); - - [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params string[] columnNames) => config.With(c => c.HideColumns(columnNames.Select(n => (IColumnHidingRule)new ColumnHidingByNameRule(n)).ToArray())); - + [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params string[] columnNames) => config.With(c => c.HideColumns(columnNames)); + [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params IColumn[] columns) => config.With(c => c.HideColumns(columns)); [PublicAPI] public static ManualConfig HideColumns(this IConfig config, params IColumnHidingRule[] rules) => config.With(c => c.HideColumns(rules)); public static ImmutableConfig CreateImmutableConfig(this IConfig config) => ImmutableConfigBuilder.Create(config); diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index fa4baee771..2f0cbd284e 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -53,6 +53,7 @@ public IEnumerable GetAnalysers() yield return RuntimeErrorAnalyser.Default; yield return ZeroMeasurementAnalyser.Default; yield return BaselineCustomAnalyzer.Default; + yield return HideColumnsAnalyser.Default; } public IEnumerable GetValidators() diff --git a/src/BenchmarkDotNet/Configs/ManualConfig.cs b/src/BenchmarkDotNet/Configs/ManualConfig.cs index f2a16b2944..30a3dfbd20 100644 --- a/src/BenchmarkDotNet/Configs/ManualConfig.cs +++ b/src/BenchmarkDotNet/Configs/ManualConfig.cs @@ -202,6 +202,20 @@ public ManualConfig AddLogicalGroupRules(params BenchmarkLogicalGroupRule[] rule return this; } + [PublicAPI] + public ManualConfig HideColumns(params string[] columnNames) + { + columnHidingRules.AddRange(columnNames.Select(c => new ColumnHidingByNameRule(c))); + return this; + } + + [PublicAPI] + public ManualConfig HideColumns(params IColumn[] columns) + { + columnHidingRules.AddRange(columns.Select(c => new ColumnHidingByIdRule(c))); + return this; + } + [PublicAPI] public ManualConfig HideColumns(params IColumnHidingRule[] rules) { diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index 0be8d39310..dd3cc8a117 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -43,6 +43,9 @@ public class CommandLineOptions [Option('f', "filter", Required = false, HelpText = "Glob patterns")] public IEnumerable Filters { get; set; } + [Option('h', "hide", Required = false, HelpText = "Hides columns by name")] + public IEnumerable HiddenColumns { get; set; } + [Option('i', "inProcess", Required = false, Default = false, HelpText = "Run benchmarks in Process")] public bool RunInProcess { get; set; } @@ -163,7 +166,7 @@ public class CommandLineOptions [Option("disableLogFile", Required = false, HelpText = "Disables the logfile.")] public bool DisableLogFile { get; set; } - [Option("maxWidth", Required = false, HelpText = "Max paramter column width, the default is 20.")] + [Option("maxWidth", Required = false, HelpText = "Max parameter column width, the default is 20.")] public int? MaxParameterColumnWidth { get; set; } [Option("envVars", Required = false, HelpText = "Colon separated environment variables (key:value)")] @@ -220,6 +223,7 @@ public static IEnumerable Examples new CommandLineOptions { Filters = new[] { "*"}, Runtimes = new[] { "netcoreapp2.0", "netcoreapp2.1" }, StatisticalTestThreshold = "5%" }); yield return new Example("Run benchmarks using environment variables 'ENV_VAR_KEY_1' with value 'value_1' and 'ENV_VAR_KEY_2' with value 'value_2'", longName, new CommandLineOptions { EnvironmentVariables = new[] { "ENV_VAR_KEY_1:value_1", "ENV_VAR_KEY_2:value_2" } }); + yield return new Example("Hide Mean and Ratio columns (use double quotes for multi-word columns: \"Alloc Ratio\")", shortName, new CommandLineOptions { HiddenColumns = new[] { "Mean", "Ratio" }, }); } } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index f1f3dbaa33..a28a179abd 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -236,6 +236,8 @@ private static IConfig CreateConfig(CommandLineOptions options, IConfig globalCo else config.AddFilter(filters); + config.HideColumns(options.HiddenColumns.ToArray()); + config.WithOption(ConfigOptions.JoinSummary, options.Join); config.WithOption(ConfigOptions.KeepBenchmarkFiles, options.KeepBenchmarkFiles); config.WithOption(ConfigOptions.DontOverwriteResults, options.DontOverwriteResults); @@ -394,7 +396,7 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma case RuntimeMoniker.WasmNet60: return MakeWasmJob(baseJob, options, timeOut, "net6.0"); case RuntimeMoniker.WasmNet70: - return MakeWasmJob(baseJob, options, timeOut, "net7.0"); + return MakeWasmJob(baseJob, options, timeOut, "net7.0"); case RuntimeMoniker.MonoAOTLLVM: return MakeMonoAOTLLVMJob(baseJob, options, timeOut, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0"); case RuntimeMoniker.MonoAOTLLVMNet60: @@ -404,25 +406,25 @@ private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, Comma default: throw new NotSupportedException($"Runtime {runtimeId} is not supported"); } - } - - private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, TimeSpan? timeOut, string msBuildMoniker) - { - var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, msBuildMoniker: msBuildMoniker); - - var toolChain = MonoAotLLVMToolChain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, - runtimeFrameworkVersion: null, - name: monoAotLLVMRuntime.Name, - customDotNetCliPath: options.CliPath?.FullName, - packagesPath: options.RestorePath?.FullName, - timeout: timeOut ?? NetCoreAppSettings.DefaultBuildTimeout, - customRuntimePack: options.CustomRuntimePack, - aotCompilerPath: options.AOTCompilerPath.ToString(), - aotCompilerMode: options.AOTCompilerMode)); - - return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain); + } + + private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, TimeSpan? timeOut, string msBuildMoniker) + { + var monoAotLLVMRuntime = new MonoAotLLVMRuntime(aotCompilerPath: options.AOTCompilerPath, msBuildMoniker: msBuildMoniker); + + var toolChain = MonoAotLLVMToolChain.From( + new NetCoreAppSettings( + targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, + runtimeFrameworkVersion: null, + name: monoAotLLVMRuntime.Name, + customDotNetCliPath: options.CliPath?.FullName, + packagesPath: options.RestorePath?.FullName, + timeout: timeOut ?? NetCoreAppSettings.DefaultBuildTimeout, + customRuntimePack: options.CustomRuntimePack, + aotCompilerPath: options.AOTCompilerPath.ToString(), + aotCompilerMode: options.AOTCompilerMode)); + + return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain); } private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, TimeSpan? timeOut, string msBuildMoniker) From 82d7b2fe7f0e0141451ec06c300c7032077acdc3 Mon Sep 17 00:00:00 2001 From: Yegor Stepanov Date: Sat, 29 Jan 2022 20:24:50 +0300 Subject: [PATCH 10/12] Print common columns when all columns are hidden --- src/BenchmarkDotNet/Exporters/MarkdownExporter.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs b/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs index 8435bbe8d7..80ae914519 100644 --- a/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs +++ b/src/BenchmarkDotNet/Exporters/MarkdownExporter.cs @@ -139,13 +139,15 @@ private void PrintTable(SummaryTable table, ILogger logger) return; } + table.PrintCommonColumns(logger); + if (table.Columns.All(c => !c.NeedToShow)) { + logger.WriteLine(); logger.WriteLine("There are no columns to show "); return; } - table.PrintCommonColumns(logger); logger.WriteLine(); if (UseCodeBlocks) From f12cda00b57ed830504a032178271cec4266cdd0 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 25 Aug 2022 14:06:09 +0200 Subject: [PATCH 11/12] solve warnings and errors --- tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs | 2 +- tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs b/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs index 2564aa5bac..94913fd1fa 100644 --- a/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs +++ b/tests/BenchmarkDotNet.Tests/Columns/MetricColumnTests.cs @@ -47,7 +47,7 @@ private static Summary CreateMockSummary(bool printUnitsInContent, bool printUni metric }); return new Summary("", new[] { benchmarkReport }.ToImmutableArray(), HostEnvironmentInfo.GetCurrent(), - "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty); + "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty, ImmutableArray.Empty); } [SuppressMessage("ReSharper", "UnassignedGetOnlyAutoProperty")] diff --git a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs index 43b588e019..04cff38e30 100644 --- a/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs +++ b/tests/BenchmarkDotNet.Tests/Order/DefaultOrdererTests.cs @@ -21,7 +21,7 @@ public class DefaultOrdererTests private static Summary CreateMockSummary() => new ("", ImmutableArray.Empty, HostEnvironmentInfo.GetCurrent(), "", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray.Empty, ImmutableArray.Empty); - private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new( + private static BenchmarkCase CreateBenchmarkCase(string category, int parameter, params BenchmarkLogicalGroupRule[] rules) => new ( new Descriptor(null, null, categories: new[] { category }), new Job(), new ParameterInstances(new[] From eb59001761b7b3dd441bd8f09a0d5d78cac5e184 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 25 Aug 2022 14:09:40 +0200 Subject: [PATCH 12/12] improvements: - don't store Names in the field when there is no need to - make it possible to apply `[HideColumnsAttribute]` to entire assembly - add comment explaining why Column class is public - make two existing column hiding rules public so they can be reused - Column.IsCommon is better name than Column.IsCommonColumn - simplify LINQ -add sample --- .../BenchmarkDotNet.Samples/IntroHidingColumns.cs | 13 +++++++++++++ .../Attributes/HideColumnsAttribute.cs | 15 +++------------ .../Columns/BaselineAllocationRatioColumn.cs | 1 + src/BenchmarkDotNet/Columns/Column.cs | 2 +- .../Columns/ColumnHidingByIdRule.cs | 2 +- .../Columns/ColumnHidingByNameRule.cs | 2 +- src/BenchmarkDotNet/Reports/SummaryTable.cs | 6 +++--- .../Reports/SummaryTableExtensions.cs | 2 +- .../Running/BenchmarkRunnerClean.cs | 6 +++--- 9 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 samples/BenchmarkDotNet.Samples/IntroHidingColumns.cs diff --git a/samples/BenchmarkDotNet.Samples/IntroHidingColumns.cs b/samples/BenchmarkDotNet.Samples/IntroHidingColumns.cs new file mode 100644 index 0000000000..127f037e9f --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/IntroHidingColumns.cs @@ -0,0 +1,13 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; + +namespace BenchmarkDotNet.Samples +{ + [MemoryDiagnoser] // adds Gen0, Gen1, Gen2 and Allocated Bytes columns + [HideColumns(Column.Gen0, Column.Gen1, Column.Gen2)] // dont display GenX columns + public class IntroHidingColumns + { + [Benchmark] + public byte[] AllocateArray() => new byte[100_000]; + } +} diff --git a/src/BenchmarkDotNet/Attributes/HideColumnsAttribute.cs b/src/BenchmarkDotNet/Attributes/HideColumnsAttribute.cs index f66350e962..587025d5e5 100644 --- a/src/BenchmarkDotNet/Attributes/HideColumnsAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/HideColumnsAttribute.cs @@ -5,23 +5,14 @@ namespace BenchmarkDotNet.Attributes { [PublicAPI] - [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] public class HideColumnsAttribute : Attribute, IConfigSource { - public string[] Names { get; } - public IConfig Config { get; } // CLS-Compliant Code requires a constructor without an array in the argument list - protected HideColumnsAttribute() - { - Config = ManualConfig.CreateEmpty(); - } + protected HideColumnsAttribute() => Config = ManualConfig.CreateEmpty(); - public HideColumnsAttribute(params string[] names) - { - Names = names; - Config = ManualConfig.CreateEmpty().HideColumns(names); - } + public HideColumnsAttribute(params string[] names) => Config = ManualConfig.CreateEmpty().HideColumns(names); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs b/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs index a7e4052ee7..c649b8c87f 100644 --- a/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs +++ b/src/BenchmarkDotNet/Columns/BaselineAllocationRatioColumn.cs @@ -12,6 +12,7 @@ namespace BenchmarkDotNet.Columns public class BaselineAllocationRatioColumn : BaselineCustomColumn { public override string Id => nameof(BaselineAllocationRatioColumn); + public override string ColumnName => Column.AllocRatio; public static readonly IColumn RatioMean = new BaselineAllocationRatioColumn(); diff --git a/src/BenchmarkDotNet/Columns/Column.cs b/src/BenchmarkDotNet/Columns/Column.cs index e9151881bd..c4207b744b 100644 --- a/src/BenchmarkDotNet/Columns/Column.cs +++ b/src/BenchmarkDotNet/Columns/Column.cs @@ -3,7 +3,7 @@ namespace BenchmarkDotNet.Columns { // ReSharper disable once InconsistentNaming - [PublicAPI] + [PublicAPI] // this type is public, so the users can do things like [HideColumns(Column.$)] and get suggestions from IDE public static class Column { public const string Namespace = "Namespace"; diff --git a/src/BenchmarkDotNet/Columns/ColumnHidingByIdRule.cs b/src/BenchmarkDotNet/Columns/ColumnHidingByIdRule.cs index 2e54665bf3..2bb89d3b47 100644 --- a/src/BenchmarkDotNet/Columns/ColumnHidingByIdRule.cs +++ b/src/BenchmarkDotNet/Columns/ColumnHidingByIdRule.cs @@ -3,7 +3,7 @@ namespace BenchmarkDotNet.Columns { [PublicAPI] - internal class ColumnHidingByIdRule: IColumnHidingRule + public class ColumnHidingByIdRule: IColumnHidingRule { public string Id { get; } diff --git a/src/BenchmarkDotNet/Columns/ColumnHidingByNameRule.cs b/src/BenchmarkDotNet/Columns/ColumnHidingByNameRule.cs index c309ab9f94..bde8e345f9 100644 --- a/src/BenchmarkDotNet/Columns/ColumnHidingByNameRule.cs +++ b/src/BenchmarkDotNet/Columns/ColumnHidingByNameRule.cs @@ -3,7 +3,7 @@ namespace BenchmarkDotNet.Columns { [PublicAPI] - internal class ColumnHidingByNameRule: IColumnHidingRule + public class ColumnHidingByNameRule: IColumnHidingRule { public string Name { get; } diff --git a/src/BenchmarkDotNet/Reports/SummaryTable.cs b/src/BenchmarkDotNet/Reports/SummaryTable.cs index bb2c411b5e..00106fbab5 100644 --- a/src/BenchmarkDotNet/Reports/SummaryTable.cs +++ b/src/BenchmarkDotNet/Reports/SummaryTable.cs @@ -108,7 +108,7 @@ public class SummaryTableColumn public TextJustification Justify { get; } public IColumn OriginalColumn { get; } - internal bool IsCommonColumn { get; } + internal bool IsCommon { get; } internal bool WasHidden { get; } public SummaryTableColumn(SummaryTable table, int index, IColumn column, bool hide = false) @@ -126,8 +126,8 @@ public SummaryTableColumn(SummaryTable table, int index, IColumn column, bool hi NeedToShow = !hide && needToShow; WasHidden = hide && needToShow; - bool isCommonColumn = !NeedToShow && !IsDefault; - IsCommonColumn = (!hide && isCommonColumn) || (hide && Content.Distinct().Count() == 1); + bool isCommon = !NeedToShow && !IsDefault; + IsCommon = (!hide && isCommon) || (hide && Content.Distinct().Count() == 1); } public override string ToString() => Header; diff --git a/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs b/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs index 3829602443..68419c3054 100644 --- a/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs +++ b/src/BenchmarkDotNet/Reports/SummaryTableExtensions.cs @@ -15,7 +15,7 @@ public static class SummaryTableExtensions public static void PrintCommonColumns(this SummaryTable table, ILogger logger) { - var commonColumns = table.Columns.Where(c => c.IsCommonColumn).ToArray(); + var commonColumns = table.Columns.Where(c => c.IsCommon).ToArray(); if (commonColumns.Any()) { int paramsOnLine = 0; diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index f27853b1c4..7e5b56c118 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -228,7 +228,7 @@ private static Summary Run(BenchmarkRunInfo benchmarkRunInfo, runEnd.GetTimeSpan() - runStart.GetTimeSpan(), cultureInfo, Validate(new[] {benchmarkRunInfo }, NullLogger.Instance), // validate them once again, but don't print the output - benchmarkRunInfo.Config.GetColumnHidingRules().ToImmutableArray()); + config.GetColumnHidingRules().ToImmutableArray()); } private static void PrintSummary(ILogger logger, ImmutableConfig config, Summary summary) @@ -264,9 +264,9 @@ private static void PrintSummary(ILogger logger, ImmutableConfig config, Summary } // TODO: move to conclusions - var columnWithLegends = summary.Table.Columns.Where(c => c.NeedToShow).Select(c => c.OriginalColumn).Where(c => !string.IsNullOrEmpty(c.Legend)).ToList(); + var columnWithLegends = summary.Table.Columns.Where(c => c.NeedToShow && !string.IsNullOrEmpty(c.OriginalColumn.Legend)).Select(c => c.OriginalColumn).ToArray(); - bool needToShowTimeLegend = summary.Table.Columns.Where(c => c.NeedToShow).Select(c => c.OriginalColumn).Any(c => c.UnitType == UnitType.Time); + bool needToShowTimeLegend = summary.Table.Columns.Any(c => c.NeedToShow && c.OriginalColumn.UnitType == UnitType.Time); var effectiveTimeUnit = needToShowTimeLegend ? summary.Table.EffectiveSummaryStyle.TimeUnit : null; if (columnWithLegends.Any() || effectiveTimeUnit != null)