-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ConfigExtensions: Tolerate both implementation and breaking API chang…
…es in BDN-nightly > 0.13.1. - Tolerate implementation changes in ManualConfig.GetXxx() Closes #25. - Tolerate breaking API changes in BenchmarkReport.ctor() and others. See #26. - Unpin NightlyBDN (Issue resolved dotnet/BenchmarkDotNet#1922). - Add tests.
- Loading branch information
Showing
6 changed files
with
770 additions
and
32 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
221 changes: 221 additions & 0 deletions
221
tests/Mawosoft.Extensions.BenchmarkDotNet.Tests/ColumnCategoryExtensionsTests.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,221 @@ | ||
| // Copyright (c) 2021-2022 Matthias Wolf, Mawosoft. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using BenchmarkDotNet.Columns; | ||
| using Xunit; | ||
| using Xunit.Abstractions; | ||
|
|
||
| using static Mawosoft.Extensions.BenchmarkDotNet.ColumnCategoryExtensions; | ||
|
|
||
| namespace Mawosoft.Extensions.BenchmarkDotNet.Tests | ||
| { | ||
| public class ColumnCategoryExtensionsTests | ||
| { | ||
| [Fact] | ||
| public void ToExtended_ExistingCategoriesMatch() | ||
| { | ||
| string[] categoryNamesArray = Enum.GetNames(typeof(ColumnCategory)); | ||
| string[] extendedNamesArray = Enum.GetNames(typeof(ExtendedColumnCategory)); | ||
| HashSet<string> categoryNames = new(categoryNamesArray); | ||
| HashSet<string> extendedNames = new(extendedNamesArray); | ||
| Assert.Equal(categoryNamesArray.Length, categoryNames.Count); | ||
| Assert.Equal(extendedNamesArray.Length, extendedNames.Count); | ||
| Assert.ProperSuperset(categoryNames, extendedNames); | ||
| foreach (string cname in categoryNames) | ||
| { | ||
| Assert.Equal( | ||
| (ExtendedColumnCategory)Enum.Parse(typeof(ExtendedColumnCategory), cname), | ||
| ColumnCategoryExtensions.ToExtended( | ||
| (ColumnCategory)Enum.Parse(typeof(ColumnCategory), cname))); | ||
| } | ||
| } | ||
|
|
||
| [Fact] | ||
| public void ToExtended_NonExistingCategories_Unknown() | ||
| { | ||
| Assert.Equal( | ||
| ExtendedColumnCategory.Unknown, | ||
| ColumnCategoryExtensions.ToExtended(ColumnCategory.Metric + 1)); | ||
| } | ||
|
|
||
| private class GetExtendedColumnCategory_TheoryData : TheoryData<IColumn, ExtendedColumnCategory> | ||
| { | ||
| // We get all existing IColumn classes via reflection and handle them by name | ||
| // to discover any new additions not covered by test. | ||
| public GetExtendedColumnCategory_TheoryData() | ||
| { | ||
| Func<Type, bool> predicate = t => t.IsClass && !t.IsAbstract && typeof(IColumn).IsAssignableFrom(t); | ||
|
|
||
| IEnumerable<Type> columnTypes = | ||
| typeof(IColumn).Assembly.GetTypes().Where(predicate) | ||
| .Concat(typeof(ColumnCategoryExtensions).Assembly.GetTypes().Where(predicate) | ||
| .Distinct()); | ||
|
|
||
| List<string> unexpectedTypes = new(); | ||
|
|
||
| foreach (Type type in columnTypes) | ||
| { | ||
| switch (type.Name) | ||
| { | ||
| // BenchmarkDotNet 0.13.1 | ||
| case "BaselineColumn": | ||
| Add(new BaselineColumn(), ExtendedColumnCategory.Meta); | ||
| break; | ||
| case "BaselineRatioColumn": | ||
| Add(BaselineRatioColumn.RatioMean, ExtendedColumnCategory.Baseline); | ||
| break; | ||
| case "BaselineScaledColumn": | ||
| #pragma warning disable CS0618 // Type or member is obsolete | ||
| Add(BaselineScaledColumn.Scaled, ExtendedColumnCategory.Baseline); | ||
| #pragma warning restore CS0618 // Type or member is obsolete | ||
| break; | ||
| case "CategoriesColumn": | ||
| Add(new CategoriesColumn(), ExtendedColumnCategory.Category); | ||
| break; | ||
| case "JobCharacteristicColumn": | ||
| Add(JobCharacteristicColumn.AllColumns[0], ExtendedColumnCategory.Job); | ||
| break; | ||
| case "LogicalGroupColumn": | ||
| Add(new LogicalGroupColumn(), ExtendedColumnCategory.Meta); | ||
| break; | ||
| case "MetricColumn": | ||
| Add(new MetricColumn(null), ExtendedColumnCategory.Metric); | ||
| break; | ||
| case "ParamColumn": | ||
| Add(new ParamColumn("head"), ExtendedColumnCategory.Params); | ||
| break; | ||
| case "RankColumn": | ||
| Add(RankColumn.Arabic, ExtendedColumnCategory.Custom); | ||
| break; | ||
| case "StatisticalTestColumn": | ||
| Add(new StatisticalTestColumn(Perfolizer.Mathematics.SignificanceTesting.StatisticalTestKind.Welch, Perfolizer.Mathematics.Thresholds.Threshold.Create(Perfolizer.Mathematics.Thresholds.ThresholdUnit.Ratio, 0), true), ExtendedColumnCategory.Baseline); | ||
| break; | ||
| case "StatisticColumn": | ||
| Add(StatisticColumn.Mean, ExtendedColumnCategory.Statistics); | ||
| break; | ||
| case "TagColumn": | ||
| Add(new TagColumn("tag", p => "tag"), ExtendedColumnCategory.Custom); | ||
| break; | ||
| case "TargetMethodColumn": | ||
| Add(TargetMethodColumn.Namespace, ExtendedColumnCategory.TargetMethod); | ||
| Add(TargetMethodColumn.Type, ExtendedColumnCategory.TargetMethod); | ||
| Add(TargetMethodColumn.Method, ExtendedColumnCategory.TargetMethod); | ||
| break; | ||
| // BenchmarkDotNet 0.13.1.x-nightly | ||
| case "BaselineAllocationRatioColumn": | ||
| IColumn barc = (Activator.CreateInstance(type) as IColumn)!; | ||
| Assert.NotNull(barc); | ||
| Add(barc, ExtendedColumnCategory.Metric); | ||
| break; | ||
| // Mawosoft.Extensions.BenchmarkDotNet | ||
| case "CombinedParamsColumn": | ||
| Add(new CombinedParamsColumn(), ExtendedColumnCategory.Params); | ||
| break; | ||
| case "JobCharacteristicColumnWithLegend": | ||
| Add(new JobCharacteristicColumnWithLegend(JobCharacteristicColumn.AllColumns[0], "legend"), ExtendedColumnCategory.Job); | ||
| break; | ||
| case "RecyclableParamColumn": | ||
| Add(new RecyclableParamColumn(1, "param", false), ExtendedColumnCategory.Params); | ||
| break; | ||
| default: | ||
| unexpectedTypes.Add(type.Name); | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| Assert.Empty(unexpectedTypes); | ||
| } | ||
| } | ||
|
|
||
| [Theory] | ||
| [ClassData(typeof(GetExtendedColumnCategory_TheoryData))] | ||
| internal void GetExtendedColumnCategory_Succeeds(IColumn column, ExtendedColumnCategory extendedColumnCategory) | ||
| { | ||
| Assert.Equal(extendedColumnCategory, ColumnCategoryExtensions.GetExtendedColumnCategory(column)); | ||
| } | ||
|
|
||
| private class GetExtendedColumnCategories_TheoryData : TheoryData<IColumnProvider, IEnumerable<ExtendedColumnCategory>> | ||
| { | ||
| // We get all existing IColumnProvider classes via reflection and handle them by name | ||
| // to discover any new additions not covered by test. | ||
| public GetExtendedColumnCategories_TheoryData() | ||
| { | ||
| Func<Type, bool> predicate = t => t.IsClass && !t.IsAbstract && typeof(IColumnProvider).IsAssignableFrom(t); | ||
|
|
||
| IEnumerable<Type> providerTypes = | ||
| typeof(IColumn).Assembly.GetTypes().Where(predicate) | ||
| .Concat(typeof(ColumnCategoryExtensions).Assembly.GetTypes().Where(predicate) | ||
| .Distinct()); | ||
|
|
||
| List<string> unexpectedTypes = new(); | ||
|
|
||
| foreach (Type type in providerTypes) | ||
| { | ||
| switch (type.Name) | ||
| { | ||
| // BenchmarkDotNet | ||
| case "CompositeColumnProvider": | ||
| Add(new CompositeColumnProvider( | ||
| DefaultColumnProviders.Descriptor, DefaultColumnProviders.Job), | ||
| new[] { ExtendedColumnCategory.TargetMethod, ExtendedColumnCategory.Job }); | ||
| Add(new CompositeColumnProvider( | ||
| new JobColumnSelectionProvider("-all +job", true), new RecyclableParamsColumnProvider()), | ||
| new[] { ExtendedColumnCategory.Job, ExtendedColumnCategory.Params }); | ||
| break; | ||
| case "EmptyColumnProvider": | ||
| Add(EmptyColumnProvider.Instance, Array.Empty<ExtendedColumnCategory>()); | ||
| break; | ||
| case "SimpleColumnProvider": | ||
| Add(new SimpleColumnProvider( | ||
| StatisticColumn.Mean, StatisticColumn.Error, TargetMethodColumn.Method), | ||
| new[] { ExtendedColumnCategory.Statistics, ExtendedColumnCategory.TargetMethod }); | ||
| Add(new SimpleColumnProvider(new CategoriesColumn()), | ||
| new[] { ExtendedColumnCategory.Category }); | ||
| break; | ||
| case "DescriptorColumnProvider": | ||
| Add(DefaultColumnProviders.Descriptor, new[] { ExtendedColumnCategory.TargetMethod }); | ||
| break; | ||
| case "JobColumnProvider": | ||
| Add(DefaultColumnProviders.Job, new[] { ExtendedColumnCategory.Job }); | ||
| break; | ||
| case "StatisticsColumnProvider": | ||
| Add(DefaultColumnProviders.Statistics, new[] { ExtendedColumnCategory.Statistics }); | ||
| break; | ||
| case "ParamsColumnProvider": | ||
| Add(DefaultColumnProviders.Params, new[] { ExtendedColumnCategory.Params }); | ||
| break; | ||
| case "MetricsColumnProvider": | ||
| Add(DefaultColumnProviders.Metrics, new[] { ExtendedColumnCategory.Metric }); | ||
| break; | ||
| // Mawosoft.Extensions.BenchmarkDotNet | ||
| case "JobColumnSelectionProvider": | ||
| Add(new JobColumnSelectionProvider("+all", true), new[] { ExtendedColumnCategory.Job }); | ||
| break; | ||
| case "RecyclableParamsColumnProvider": | ||
| Add(new RecyclableParamsColumnProvider(), new[] { ExtendedColumnCategory.Params }); | ||
| break; | ||
| default: | ||
| unexpectedTypes.Add(type.Name); | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| Assert.Empty(unexpectedTypes); | ||
| } | ||
| } | ||
|
|
||
| [Theory] | ||
| [ClassData(typeof(GetExtendedColumnCategories_TheoryData))] | ||
| internal void GetExtendedColumnCategories_Succeeds(IColumnProvider provider, IEnumerable<ExtendedColumnCategory> extendedColumnCategories) | ||
| { | ||
| Assert.Equal( | ||
| new HashSet<ExtendedColumnCategory>(extendedColumnCategories), | ||
| new HashSet<ExtendedColumnCategory>(ColumnCategoryExtensions.GetExtendedColumnCategories(provider))); | ||
| } | ||
|
|
||
| // For ColumnCategoryExtensions.RemoveColumnsByCategory see RemoveColumnsByCategory_Succeeds | ||
| // in ConfigExtensionsTests.Columns.cs | ||
| } | ||
| } |
Oops, something went wrong.