diff --git a/.gitignore b/.gitignore index 0ab3e391..31bf65c0 100644 --- a/.gitignore +++ b/.gitignore @@ -558,6 +558,7 @@ typings/ # Specific ones added coverage-outputs/** .env +tmp/** ### Rules Framework Web UI ### !src/Rules.Framework.WebUI/node_modules/ diff --git a/run-benchmarks.ps1 b/run-benchmarks.ps1 new file mode 100644 index 00000000..f3e03a0b --- /dev/null +++ b/run-benchmarks.ps1 @@ -0,0 +1,55 @@ +param ([switch] $KeepBenchmarksFiles) + +$originalDir = (Get-Location).Path +$timestamp = [DateTime]::UtcNow.ToString("yyyyMMdd_hhmmss") + +$directoryFound = Get-ChildItem -Path $originalDir -Directory | Select-String -Pattern "tmp" +if (!$directoryFound) { + New-Item -Name "tmp" -ItemType Directory > $null +} + +$directoryFound = Get-ChildItem -Path "$originalDir\\tmp" -Directory | Select-String -Pattern "benchmarks" +if (!$directoryFound) { + New-Item -Name "benchmarks" -ItemType Directory -Path "$originalDir\\tmp" > $null +} + +$directoryFound = Get-ChildItem -Path "$originalDir\\tmp\\benchmarks" -Directory | Select-String -Pattern $timestamp +if (!$directoryFound) { + New-Item -Name $timestamp -ItemType Directory -Path "$originalDir\\tmp\\benchmarks" > $null +} + +$reportDir = "$originalDir\\tmp\\benchmarks\\$timestamp" + +# Ensure all packages restored before running benchmarks +dotnet restore rules-framework.sln + +# Build benchmarks binaries +dotnet build -c Release .\tests\Rules.Framework.BenchmarkTests\Rules.Framework.BenchmarkTests.csproj -o "$reportDir\\bin" --framework net6.0 + +Set-Location -Path $reportDir + +# Run benchmarks +bin\Rules.Framework.BenchmarkTests.exe -a artifacts + +# Determine results file +$filteredResultsFiles = Get-ChildItem -Path artifacts/results -File -Filter *.md +if ($filteredResultsFiles) { + $resultsFile = $filteredResultsFiles.Name + + # Copy results file + Copy-Item -Path artifacts/results/$resultsFile -Destination . + + # Rename file + Rename-Item -Path $resultsFile -NewName report.md +} + +if (!$KeepBenchmarksFiles) { + if ($directoryFound = Get-ChildItem -Path $reportDir -Directory | Select-String -Pattern "artifacts") { + Remove-Item -Path artifacts -Recurse > $null + } + if ($directoryFound = Get-ChildItem -Path $reportDir -Directory | Select-String -Pattern "bin") { + Remove-Item -Path bin -Recurse > $null + } +} + +Set-Location -Path $originalDir \ No newline at end of file diff --git a/src/Rules.Framework/Core/ConditionNodes/ComposedConditionNode.cs b/src/Rules.Framework/Core/ConditionNodes/ComposedConditionNode.cs index 622eef39..7f56cb0d 100644 --- a/src/Rules.Framework/Core/ConditionNodes/ComposedConditionNode.cs +++ b/src/Rules.Framework/Core/ConditionNodes/ComposedConditionNode.cs @@ -2,6 +2,7 @@ namespace Rules.Framework.Core.ConditionNodes { using System; using System.Collections.Generic; + using System.Collections.ObjectModel; using System.Diagnostics; using System.Linq; diff --git a/tests/Rules.Framework.BenchmarkTests/CustomBaselineClassifierColumn.cs b/tests/Rules.Framework.BenchmarkTests/CustomBaselineClassifierColumn.cs new file mode 100644 index 00000000..4cd312b8 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/CustomBaselineClassifierColumn.cs @@ -0,0 +1,36 @@ +namespace Rules.Framework.BenchmarkTests +{ + using System; + using BenchmarkDotNet.Columns; + using BenchmarkDotNet.Reports; + using BenchmarkDotNet.Running; + + internal class CustomBaselineClassifierColumn : IColumn + { + private readonly Func classifyLogicFunc; + + public CustomBaselineClassifierColumn(Func classifyLogicFunc) + { + this.classifyLogicFunc = classifyLogicFunc; + } + + public bool AlwaysShow => true; + public ColumnCategory Category => ColumnCategory.Baseline; + public string ColumnName => "Baseline"; + public string Id => nameof(CustomBaselineClassifierColumn); + public bool IsNumeric => false; + public string Legend => "Sets wether a test case is a baseline."; + public int PriorityInCategory => 0; + public UnitType UnitType => UnitType.Dimensionless; + + public string GetValue(Summary summary, BenchmarkCase benchmarkCase) + => this.classifyLogicFunc.Invoke(benchmarkCase) ? "Yes" : "No"; + + public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) + => this.GetValue(summary, benchmarkCase); + + public bool IsAvailable(Summary summary) => true; + + public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false; + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkReport.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkReport.cs new file mode 100644 index 00000000..764799a0 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkReport.cs @@ -0,0 +1,17 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Json +{ + using System; + + internal class BenchmarkReport + { + public DateTime Date { get; set; } + + public Environment? Environment { get; set; } + + public IEnumerable? Statistics { get; set; } + + public IEnumerable? StatisticsComparison { get; set; } + + public string Title { get; set; } = string.Empty; + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsComparisonItem.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsComparisonItem.cs new file mode 100644 index 00000000..0c5c26e6 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsComparisonItem.cs @@ -0,0 +1,23 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Json +{ + internal class BenchmarkStatisticsComparisonItem + { + public BenchmarkStatisticsValue? AllocatedMemoryRate { get; set; } + + public BenchmarkStatisticsValue? BaselineAllocatedMemory { get; set; } + + public BenchmarkStatisticsValue? BaselineMeanTimeTaken { get; set; } + + public string BaselineParameters { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? CompareAllocatedMemory { get; set; } + + public BenchmarkStatisticsValue? CompareMeanTimeTaken { get; set; } + + public string CompareParameters { get; set; } = string.Empty; + + public string Key { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? MeanTimeTakenCompareRate { get; set; } + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsItem.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsItem.cs new file mode 100644 index 00000000..343ea4c4 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsItem.cs @@ -0,0 +1,23 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Json +{ + internal class BenchmarkStatisticsItem + { + public BenchmarkStatisticsValue? AllocatedMemory { get; set; } + + public string Baseline { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? BranchInstructionsPerOp { get; set; } + + public BenchmarkStatisticsValue? BranchMispredictionsPerOp { get; set; } + + public BenchmarkStatisticsValue? Gen0Collects { get; set; } + + public string Key { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? MeanTimeTaken { get; set; } + + public string Parameters { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? StandardError { get; set; } + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsValue.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsValue.cs new file mode 100644 index 00000000..c487752e --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsValue.cs @@ -0,0 +1,11 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Json +{ + internal class BenchmarkStatisticsValue + { + public string Format { get; set; } = string.Empty; + + public string Unit { get; set; } = string.Empty; + + public decimal Value { get; set; } + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Json/CustomJsonExporter.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/CustomJsonExporter.cs new file mode 100644 index 00000000..fb43a89d --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/CustomJsonExporter.cs @@ -0,0 +1,167 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Json +{ + using System.Collections.Generic; + using BenchmarkDotNet.Exporters; + using BenchmarkDotNet.Loggers; + using BenchmarkDotNet.Reports; + using Newtonsoft.Json; + + internal class CustomJsonExporter : ExporterBase + { + public CustomJsonExporter(bool indentJson) + { + this.IndentJson = indentJson; + } + + public static IExporter Compressed => new CustomJsonExporter(indentJson: false); + + public static IExporter Default => Indented; + + public static IExporter Indented => new CustomJsonExporter(indentJson: true); + + protected override string FileExtension => "json"; + + private bool IndentJson { get; } + + public override void ExportToLog(Summary summary, ILogger logger) + { + var report = CustomJsonExporter.CreateReport(summary); + + var settings = new JsonSerializerSettings + { + Formatting = Formatting.None + }; + + if (this.IndentJson) + { + settings.Formatting = Formatting.Indented; + } + + string jsonText = JsonConvert.SerializeObject(report, settings); + + logger.WriteLine(jsonText); + } + + private static decimal CalculateRate(decimal baselineValue, decimal compareValue) + => ((baselineValue - compareValue) / baselineValue) * 100; + + private static BenchmarkStatisticsComparisonItem CreateBenchmarkStatisticsComparisonItem(BenchmarkStatisticsItem? baselineStatisticsItem, BenchmarkStatisticsItem? nonBaselineStatisticsItem) => new BenchmarkStatisticsComparisonItem + { + AllocatedMemoryRate = new BenchmarkStatisticsValue + { + Format = "0.##", + Unit = "%", + Value = CalculateRate( + baselineStatisticsItem.AllocatedMemory?.Value ?? 0, + nonBaselineStatisticsItem.AllocatedMemory?.Value ?? 0), + }, + BaselineAllocatedMemory = baselineStatisticsItem.AllocatedMemory, + BaselineMeanTimeTaken = baselineStatisticsItem.MeanTimeTaken, + BaselineParameters = baselineStatisticsItem.Parameters, + CompareAllocatedMemory = nonBaselineStatisticsItem.AllocatedMemory, + CompareMeanTimeTaken = nonBaselineStatisticsItem.MeanTimeTaken, + CompareParameters = nonBaselineStatisticsItem.Parameters, + Key = baselineStatisticsItem.Key, + MeanTimeTakenCompareRate = new BenchmarkStatisticsValue + { + Format = "0.##", + Unit = "%", + Value = CalculateRate( + baselineStatisticsItem.MeanTimeTaken?.Value ?? 0, + nonBaselineStatisticsItem.MeanTimeTaken?.Value ?? 0), + } + }; + + private static BenchmarkStatisticsItem CreateBenchmarkStatisticsItem(SummaryTable.SummaryTableColumn baselineClassifierColumn, ref int current, BenchmarkDotNet.Reports.BenchmarkReport benchmarkReport) + { + var allocatedMemoryMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "Allocated Memory"); + var branchInstructionsMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "BranchInstructions"); + var branchMispredictionsMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "BranchMispredictions"); + var gen0CollectsMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "Gen0Collects"); + + var statisticsItem = new BenchmarkStatisticsItem + { + AllocatedMemory = allocatedMemoryMetric.Key is null ? null : CreateBenchmarkStatisticsValue(allocatedMemoryMetric), + Baseline = baselineClassifierColumn.Content[current++], + BranchInstructionsPerOp = + branchInstructionsMetric.Key is null ? null : CreateBenchmarkStatisticsValue(branchInstructionsMetric), + BranchMispredictionsPerOp = + branchMispredictionsMetric.Key is null ? null : CreateBenchmarkStatisticsValue(branchMispredictionsMetric), + Gen0Collects = gen0CollectsMetric.Key is null ? null : CreateBenchmarkStatisticsValue(gen0CollectsMetric), + Key = $"{benchmarkReport.BenchmarkCase.Descriptor.Type.Name}.{benchmarkReport.BenchmarkCase.Descriptor.WorkloadMethod.Name}", + MeanTimeTaken = new BenchmarkStatisticsValue + { + Format = "N0", + Unit = "ns", + Value = Convert.ToDecimal(benchmarkReport.ResultStatistics?.Mean ?? 0), + }, + Parameters = benchmarkReport.BenchmarkCase.Parameters.DisplayInfo, + StandardError = new BenchmarkStatisticsValue + { + Format = "N0", + Unit = "ns", + Value = Convert.ToDecimal(benchmarkReport.ResultStatistics?.StandardError ?? 0), + }, + }; + return statisticsItem; + } + + private static BenchmarkStatisticsValue CreateBenchmarkStatisticsValue(KeyValuePair allocatedMemoryMetric) => new BenchmarkStatisticsValue + { + Format = allocatedMemoryMetric.Value.Descriptor.NumberFormat, + Unit = allocatedMemoryMetric.Value.Descriptor.Unit, + Value = Convert.ToDecimal(allocatedMemoryMetric.Value.Value), + }; + + private static Environment CreateEnvironment(Summary summary) => new Environment + { + Architecture = summary.HostEnvironmentInfo.Architecture, + BenchmarkDotNetCaption = "BenchmarkDotNet", + BenchmarkDotNetVersion = summary.HostEnvironmentInfo.BenchmarkDotNetVersion, + BuildConfiguration = summary.HostEnvironmentInfo.Configuration, + DotNetCliVersion = summary.HostEnvironmentInfo.DotNetSdkVersion.Value, + DotNetRuntimeVersion = summary.HostEnvironmentInfo.RuntimeVersion, + LogicalCoreCount = summary.HostEnvironmentInfo.CpuInfo.Value.LogicalCoreCount.GetValueOrDefault(0), + PhysicalCoreCount = summary.HostEnvironmentInfo.CpuInfo.Value.PhysicalCoreCount.GetValueOrDefault(0), + ProcessorName = summary.HostEnvironmentInfo.CpuInfo.Value.ProcessorName, + }; + + private static BenchmarkReport CreateReport(Summary summary) + { + var report = new BenchmarkReport + { + Date = DateTime.UtcNow, + Environment = CreateEnvironment(summary), + Title = summary.Title, + }; + var statistics = new List(summary.Reports.Length); + + var baselineClassifierColumn = summary.Table.Columns.First(c => c.OriginalColumn.ColumnName == "Baseline"); + var current = 0; + foreach (var benchmarkReport in summary.Reports) + { + var statisticsItem = CreateBenchmarkStatisticsItem(baselineClassifierColumn, ref current, benchmarkReport); + + statistics.Add(statisticsItem); + } + report.Statistics = statistics; + + var baselineStatisticsItems = statistics.Where(i => string.Equals(i.Baseline, "Yes")); + var nonBaselineStatisticsItems = statistics.Where(i => string.Equals(i.Baseline, "No")); + var statisticsComparison = new List(); + + foreach (var baselineStatisticsItem in baselineStatisticsItems) + { + foreach (var nonBaselineStatisticsItem in nonBaselineStatisticsItems.Where(i => string.Equals(i.Key, baselineStatisticsItem.Key))) + { + var statisticsComparisonItem = CreateBenchmarkStatisticsComparisonItem(baselineStatisticsItem, nonBaselineStatisticsItem); + + statisticsComparison.Add(statisticsComparisonItem); + } + } + report.StatisticsComparison = statisticsComparison; + + return report; + } + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Json/Environment.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/Environment.cs new file mode 100644 index 00000000..664edd42 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Json/Environment.cs @@ -0,0 +1,23 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Json +{ + internal class Environment + { + public string Architecture { get; set; } = string.Empty; + + public string BenchmarkDotNetCaption { get; set; } = string.Empty; + + public string BenchmarkDotNetVersion { get; set; } = string.Empty; + + public string BuildConfiguration { get; set; } = string.Empty; + + public string DotNetCliVersion { get; set; } = string.Empty; + + public string DotNetRuntimeVersion { get; set; } = string.Empty; + + public int LogicalCoreCount { get; set; } + + public int PhysicalCoreCount { get; set; } + + public string ProcessorName { get; set; } = string.Empty; + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkReport.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkReport.cs new file mode 100644 index 00000000..5a14e963 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkReport.cs @@ -0,0 +1,17 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Markdown +{ + using System; + + internal class BenchmarkReport + { + public DateTime Date { get; set; } + + public Environment? Environment { get; set; } + + public IEnumerable? Statistics { get; set; } + + public IEnumerable? StatisticsComparison { get; set; } + + public string Title { get; set; } = string.Empty; + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkStatisticsComparisonItem.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkStatisticsComparisonItem.cs new file mode 100644 index 00000000..8d96bff5 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkStatisticsComparisonItem.cs @@ -0,0 +1,23 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Markdown +{ + internal class BenchmarkStatisticsComparisonItem + { + public BenchmarkStatisticsValue? AllocatedMemoryRate { get; set; } + + public BenchmarkStatisticsValue? BaselineAllocatedMemory { get; set; } + + public BenchmarkStatisticsValue? BaselineMeanTimeTaken { get; set; } + + public string BaselineParameters { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? CompareAllocatedMemory { get; set; } + + public BenchmarkStatisticsValue? CompareMeanTimeTaken { get; set; } + + public string CompareParameters { get; set; } = string.Empty; + + public string Key { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? MeanTimeTakenCompareRate { get; set; } + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkStatisticsItem.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkStatisticsItem.cs new file mode 100644 index 00000000..fcb02d84 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkStatisticsItem.cs @@ -0,0 +1,23 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Markdown +{ + internal class BenchmarkStatisticsItem + { + public BenchmarkStatisticsValue? AllocatedMemory { get; set; } + + public string Baseline { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? BranchInstructionsPerOp { get; set; } + + public BenchmarkStatisticsValue? BranchMispredictionsPerOp { get; set; } + + public BenchmarkStatisticsValue? Gen0Collects { get; set; } + + public string Key { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? MeanTimeTaken { get; set; } + + public string Parameters { get; set; } = string.Empty; + + public BenchmarkStatisticsValue? StandardError { get; set; } + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkStatisticsValue.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkStatisticsValue.cs new file mode 100644 index 00000000..3b4d8080 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/BenchmarkStatisticsValue.cs @@ -0,0 +1,11 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Markdown +{ + internal class BenchmarkStatisticsValue + { + public string Format { get; set; } = string.Empty; + + public string Unit { get; set; } = string.Empty; + + public decimal Value { get; set; } + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/CustomMarkdownExporter.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/CustomMarkdownExporter.cs new file mode 100644 index 00000000..8056e89c --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/CustomMarkdownExporter.cs @@ -0,0 +1,227 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Markdown +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Text; + using BenchmarkDotNet.Exporters; + using BenchmarkDotNet.Loggers; + using BenchmarkDotNet.Reports; + + internal class CustomMarkdownExporter : ExporterBase + { + public static IExporter Default => new CustomMarkdownExporter(); + + protected override string FileExtension => "md"; + + public override void ExportToLog(Summary summary, ILogger logger) + { + var report = CustomMarkdownExporter.CreateReport(summary); + + string markdownReport = CustomMarkdownExporter.ParseReportAsMarkdown(report); + + logger.WriteLine(markdownReport); + } + + private static decimal CalculateRate(decimal baselineValue, decimal compareValue) + => ((baselineValue - compareValue) / baselineValue) * 100; + + private static BenchmarkStatisticsComparisonItem CreateBenchmarkStatisticsComparisonItem(BenchmarkStatisticsItem? baselineStatisticsItem, BenchmarkStatisticsItem? nonBaselineStatisticsItem) => new BenchmarkStatisticsComparisonItem + { + AllocatedMemoryRate = new BenchmarkStatisticsValue + { + Format = "0.##", + Unit = "%", + Value = CalculateRate( + baselineStatisticsItem.AllocatedMemory?.Value ?? 0, + nonBaselineStatisticsItem.AllocatedMemory?.Value ?? 0), + }, + BaselineAllocatedMemory = baselineStatisticsItem.AllocatedMemory, + BaselineMeanTimeTaken = baselineStatisticsItem.MeanTimeTaken, + BaselineParameters = baselineStatisticsItem.Parameters, + CompareAllocatedMemory = nonBaselineStatisticsItem.AllocatedMemory, + CompareMeanTimeTaken = nonBaselineStatisticsItem.MeanTimeTaken, + CompareParameters = nonBaselineStatisticsItem.Parameters, + Key = baselineStatisticsItem.Key, + MeanTimeTakenCompareRate = new BenchmarkStatisticsValue + { + Format = "0.##", + Unit = "%", + Value = CalculateRate( + baselineStatisticsItem.MeanTimeTaken?.Value ?? 0, + nonBaselineStatisticsItem.MeanTimeTaken?.Value ?? 0), + } + }; + + private static BenchmarkStatisticsItem CreateBenchmarkStatisticsItem(SummaryTable.SummaryTableColumn baselineClassifierColumn, ref int current, BenchmarkDotNet.Reports.BenchmarkReport benchmarkReport) + { + var allocatedMemoryMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "Allocated Memory"); + var branchInstructionsMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "BranchInstructions"); + var branchMispredictionsMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "BranchMispredictions"); + var gen0CollectsMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "Gen0Collects"); + + var statisticsItem = new BenchmarkStatisticsItem + { + AllocatedMemory = allocatedMemoryMetric.Key is null ? null : CreateBenchmarkStatisticsValue(allocatedMemoryMetric), + Baseline = baselineClassifierColumn.Content[current++], + BranchInstructionsPerOp = + branchInstructionsMetric.Key is null ? null : CreateBenchmarkStatisticsValue(branchInstructionsMetric), + BranchMispredictionsPerOp = + branchMispredictionsMetric.Key is null ? null : CreateBenchmarkStatisticsValue(branchMispredictionsMetric), + Gen0Collects = gen0CollectsMetric.Key is null ? null : CreateBenchmarkStatisticsValue(gen0CollectsMetric), + Key = $"{benchmarkReport.BenchmarkCase.Descriptor.Type.Name}.{benchmarkReport.BenchmarkCase.Descriptor.WorkloadMethod.Name}", + MeanTimeTaken = new BenchmarkStatisticsValue + { + Format = "N0", + Unit = "ns", + Value = Convert.ToDecimal(benchmarkReport.ResultStatistics?.Mean ?? 0), + }, + Parameters = benchmarkReport.BenchmarkCase.Parameters.DisplayInfo, + StandardError = new BenchmarkStatisticsValue + { + Format = "N0", + Unit = "ns", + Value = Convert.ToDecimal(benchmarkReport.ResultStatistics?.StandardError ?? 0), + }, + }; + return statisticsItem; + } + + private static BenchmarkStatisticsValue CreateBenchmarkStatisticsValue(KeyValuePair allocatedMemoryMetric) => new BenchmarkStatisticsValue + { + Format = allocatedMemoryMetric.Value.Descriptor.NumberFormat, + Unit = allocatedMemoryMetric.Value.Descriptor.Unit, + Value = Convert.ToDecimal(allocatedMemoryMetric.Value.Value), + }; + + private static Environment CreateEnvironment(Summary summary) => new Environment + { + Architecture = summary.HostEnvironmentInfo.Architecture, + BenchmarkDotNetCaption = "BenchmarkDotNet", + BenchmarkDotNetVersion = summary.HostEnvironmentInfo.BenchmarkDotNetVersion, + BuildConfiguration = summary.HostEnvironmentInfo.Configuration, + DotNetCliVersion = summary.HostEnvironmentInfo.DotNetSdkVersion.Value, + DotNetRuntimeVersion = summary.HostEnvironmentInfo.RuntimeVersion, + LogicalCoreCount = summary.HostEnvironmentInfo.CpuInfo.Value.LogicalCoreCount.GetValueOrDefault(0), + PhysicalCoreCount = summary.HostEnvironmentInfo.CpuInfo.Value.PhysicalCoreCount.GetValueOrDefault(0), + ProcessorName = summary.HostEnvironmentInfo.CpuInfo.Value.ProcessorName, + }; + + private static BenchmarkReport CreateReport(Summary summary) + { + var report = new BenchmarkReport + { + Date = DateTime.UtcNow, + Environment = CreateEnvironment(summary), + Title = summary.Title, + }; + var statistics = new List(summary.Reports.Length); + + var baselineClassifierColumn = summary.Table.Columns.First(c => c.OriginalColumn.ColumnName == "Baseline"); + var current = 0; + foreach (var benchmarkReport in summary.Reports) + { + var statisticsItem = CreateBenchmarkStatisticsItem(baselineClassifierColumn, ref current, benchmarkReport); + + statistics.Add(statisticsItem); + } + report.Statistics = statistics; + + var baselineStatisticsItems = statistics.Where(i => string.Equals(i.Baseline, "Yes")); + var nonBaselineStatisticsItems = statistics.Where(i => string.Equals(i.Baseline, "No")); + var statisticsComparison = new List(); + + foreach (var baselineStatisticsItem in baselineStatisticsItems) + { + foreach (var nonBaselineStatisticsItem in nonBaselineStatisticsItems.Where(i => string.Equals(i.Key, baselineStatisticsItem.Key))) + { + var statisticsComparisonItem = CreateBenchmarkStatisticsComparisonItem(baselineStatisticsItem, nonBaselineStatisticsItem); + + statisticsComparison.Add(statisticsComparisonItem); + } + } + report.StatisticsComparison = statisticsComparison; + + return report; + } + + private static string ParseReportAsMarkdown(BenchmarkReport benchmarkReport) + { + var hostEnvInfo = benchmarkReport.Environment; + var stringBuilder = new StringBuilder("# Benchmark Results Report") + .AppendLine() + .AppendLine($"Date & Time: {benchmarkReport.Date:yyyy-MM-dd HH:mm:ss}") + .AppendLine() + .AppendLine("## Environment") + .AppendLine() + .AppendLine($">{hostEnvInfo.BenchmarkDotNetCaption} Version={benchmarkReport.Environment.BenchmarkDotNetVersion}") + .AppendLine(">") + .AppendLine($">Processor={hostEnvInfo.ProcessorName},{hostEnvInfo.PhysicalCoreCount} physical cores, {hostEnvInfo.LogicalCoreCount} logical cores") + .AppendLine(">") + .AppendLine($">Architecture={hostEnvInfo.Architecture}, Runtime={hostEnvInfo.DotNetRuntimeVersion}, Configuration={hostEnvInfo.BuildConfiguration}") + .AppendLine(">") + .AppendLine($">.NET CLI Version={hostEnvInfo.DotNetCliVersion}"); + + stringBuilder.AppendLine() + .AppendLine("## Statistics") + .AppendLine(); + + var hasBranchInstructionsPerOp = benchmarkReport.Statistics.FirstOrDefault()?.BranchInstructionsPerOp is not null; + var hasBranchMispredictionsPerOp = benchmarkReport.Statistics.FirstOrDefault()?.BranchMispredictionsPerOp is not null; + var hasGen0Collects = benchmarkReport.Statistics.FirstOrDefault()?.Gen0Collects is not null; + var hasAllocatedMemory = benchmarkReport.Statistics.FirstOrDefault()?.AllocatedMemory is not null; + + stringBuilder.Append("|Name|Parameters|Mean Time Taken|Std Error|") + .AppendIf(() => "Branch
Instructions/Op|", () => hasBranchInstructionsPerOp) + .AppendIf(() => "Branch
Mispredictions/Op|", () => hasBranchMispredictionsPerOp) + .AppendIf(() => "GC Gen0|", () => hasGen0Collects) + .AppendIf(() => "Allocated Memory|", () => hasAllocatedMemory) + .AppendLine() + .Append("|---|---|---|---|") + .AppendIf(() => "---|", () => hasBranchInstructionsPerOp) + .AppendIf(() => "---|", () => hasBranchMispredictionsPerOp) + .AppendIf(() => "---|", () => hasGen0Collects) + .AppendIf(() => "---|", () => hasAllocatedMemory) + .AppendLine(); + + foreach (var statisticsItem in benchmarkReport.Statistics) + { + stringBuilder.Append($"|{statisticsItem.Key}") + .Append($"|{statisticsItem.Parameters}") + .Append($"|{statisticsItem.MeanTimeTaken.Value.ToString((string)statisticsItem.MeanTimeTaken.Format)} {statisticsItem.MeanTimeTaken.Unit}") + .Append($"|{statisticsItem.StandardError.Value.ToString((string)statisticsItem.StandardError.Format)} {statisticsItem.StandardError.Unit}") + .AppendIf(() => $"|{statisticsItem.BranchInstructionsPerOp.Value.ToString((string)statisticsItem.BranchInstructionsPerOp.Format)}", () => hasBranchInstructionsPerOp) + .AppendIf(() => $"|{statisticsItem.BranchMispredictionsPerOp.Value.ToString((string)statisticsItem.BranchMispredictionsPerOp.Format)}", () => hasBranchMispredictionsPerOp) + .AppendIf(() => $"|{statisticsItem.Gen0Collects.Value.ToString((string)statisticsItem.Gen0Collects.Format)}", () => hasGen0Collects) + .AppendIf(() => $"|{statisticsItem.AllocatedMemory.Value.ToString((string)statisticsItem.AllocatedMemory.Format)} {statisticsItem.AllocatedMemory.Unit}", () => hasAllocatedMemory) + .AppendLine("|"); + } + + stringBuilder.AppendLine() + .AppendLine("## Statistics Comparison") + .AppendLine() + .Append("|Name|Baseline|Compare|Mean Time Taken
[Baseline]|Mean Time Taken
[Compare]|Mean Time Taken
[Comparison %]") + .AppendIf(() => "|Allocated Memory
[Baseline]|Allocated Memory
[Compare]|Allocated Memory
[Comparison %]", () => hasAllocatedMemory) + .AppendLine("|") + .Append("|---|---|---|---|---|---") + .AppendIf(() => "|---|---|---", () => hasAllocatedMemory) + .AppendLine("|"); + + foreach (var statisticsComparisonItem in benchmarkReport.StatisticsComparison) + { + stringBuilder.Append($"|{statisticsComparisonItem.Key}") + .Append($"|{statisticsComparisonItem.BaselineParameters}") + .Append($"|{statisticsComparisonItem.CompareParameters}") + .Append($"|{statisticsComparisonItem.BaselineMeanTimeTaken.Value.ToString((string)statisticsComparisonItem.BaselineMeanTimeTaken.Format)} {statisticsComparisonItem.BaselineMeanTimeTaken.Unit}") + .Append($"|{statisticsComparisonItem.CompareMeanTimeTaken.Value.ToString((string)statisticsComparisonItem.CompareMeanTimeTaken.Format)} {statisticsComparisonItem.CompareMeanTimeTaken.Unit}") + .Append($"|{statisticsComparisonItem.MeanTimeTakenCompareRate.Value.ToString((string)statisticsComparisonItem.MeanTimeTakenCompareRate.Format)} {statisticsComparisonItem.MeanTimeTakenCompareRate.Unit}") + .AppendIf(() => $"|{statisticsComparisonItem.BaselineAllocatedMemory.Value.ToString((string)statisticsComparisonItem.BaselineAllocatedMemory.Format)} {statisticsComparisonItem.BaselineAllocatedMemory.Unit}", () => hasAllocatedMemory) + .AppendIf(() => $"|{statisticsComparisonItem.CompareAllocatedMemory.Value.ToString((string)statisticsComparisonItem.CompareAllocatedMemory.Format)} {statisticsComparisonItem.CompareAllocatedMemory.Unit}", () => hasAllocatedMemory) + .AppendIf(() => $"|{statisticsComparisonItem.AllocatedMemoryRate.Value.ToString((string)statisticsComparisonItem.AllocatedMemoryRate.Format)} {statisticsComparisonItem.AllocatedMemoryRate.Unit}", () => hasAllocatedMemory) + .AppendLine("|"); + } + + return stringBuilder.ToString(); + } + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/Environment.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/Environment.cs new file mode 100644 index 00000000..93d72904 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/Environment.cs @@ -0,0 +1,23 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Markdown +{ + internal class Environment + { + public string Architecture { get; set; } = string.Empty; + + public string BenchmarkDotNetCaption { get; set; } = string.Empty; + + public string BenchmarkDotNetVersion { get; set; } = string.Empty; + + public string BuildConfiguration { get; set; } = string.Empty; + + public string DotNetCliVersion { get; set; } = string.Empty; + + public string DotNetRuntimeVersion { get; set; } = string.Empty; + + public int LogicalCoreCount { get; set; } + + public int PhysicalCoreCount { get; set; } + + public string ProcessorName { get; set; } = string.Empty; + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/StringBuilderExtensions.cs b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/StringBuilderExtensions.cs new file mode 100644 index 00000000..c09ccd80 --- /dev/null +++ b/tests/Rules.Framework.BenchmarkTests/Exporters/Markdown/StringBuilderExtensions.cs @@ -0,0 +1,18 @@ +namespace Rules.Framework.BenchmarkTests.Exporters.Markdown +{ + using System; + using System.Text; + + internal static class StringBuilderExtensions + { + public static StringBuilder AppendIf(this StringBuilder builder, Func textFunc, Func conditionFunc) + { + if (conditionFunc.Invoke()) + { + return builder.Append(textFunc.Invoke()); + } + + return builder; + } + } +} \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Program.cs b/tests/Rules.Framework.BenchmarkTests/Program.cs index c1a11ec0..879d8585 100644 --- a/tests/Rules.Framework.BenchmarkTests/Program.cs +++ b/tests/Rules.Framework.BenchmarkTests/Program.cs @@ -1,27 +1,59 @@ +using System.Runtime.InteropServices; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; +using McMaster.Extensions.CommandLineUtils; +using Rules.Framework.BenchmarkTests; +using Rules.Framework.BenchmarkTests.Exporters.Markdown; [assembly: SimpleJob(RuntimeMoniker.Net60)] internal static class Program { - private static void Main(string[] args) + private static int Main(string[] args) { - Console.WriteLine("Starting benchmark tests."); - Console.WriteLine(); + var app = new CommandLineApplication(); - var manualConfig = ManualConfig.CreateMinimumViable(); - manualConfig.AddDiagnoser(MemoryDiagnoser.Default); - manualConfig.AddHardwareCounters(HardwareCounter.BranchInstructions, HardwareCounter.BranchMispredictions); - manualConfig.AddExporter(HtmlExporter.Default); + app.HelpOption(); - _ = BenchmarkRunner.Run(typeof(Program).Assembly, manualConfig); + var artifactsPathOption = app.Option("-a|--artifacts-path ", "Sets the artifacts path", CommandOptionType.SingleValue, config => + { + config.DefaultValue = "artifacts"; + }); - Console.WriteLine("Press any key to exit..."); - Console.Read(); + app.OnExecute(() => + { + Console.WriteLine("Starting benchmark tests."); + Console.WriteLine(); + + var manualConfig = ManualConfig.CreateMinimumViable(); + manualConfig.AddDiagnoser(MemoryDiagnoser.Default); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + manualConfig.AddHardwareCounters(HardwareCounter.BranchInstructions, HardwareCounter.BranchMispredictions); + } + + manualConfig.AddExporter(HtmlExporter.Default); + manualConfig.AddExporter(CustomMarkdownExporter.Default); + manualConfig.WithOption(ConfigOptions.JoinSummary, true); + + var artifactsPath = artifactsPathOption.Value(); + if (artifactsPath is not null or "") + { + manualConfig.WithArtifactsPath(artifactsPath); + } + + var column = new CustomBaselineClassifierColumn( + bc => bc.Parameters.Items.Any(p => p.Name == "EnableCompilation" && (bool)p.Value == false)); + manualConfig.AddColumn(column); + + _ = BenchmarkRunner.Run(typeof(Program).Assembly, manualConfig); + }); + + return app.Execute(args); } } \ No newline at end of file diff --git a/tests/Rules.Framework.BenchmarkTests/Rules.Framework.BenchmarkTests.csproj b/tests/Rules.Framework.BenchmarkTests/Rules.Framework.BenchmarkTests.csproj index 22612f9f..99ea90e9 100644 --- a/tests/Rules.Framework.BenchmarkTests/Rules.Framework.BenchmarkTests.csproj +++ b/tests/Rules.Framework.BenchmarkTests/Rules.Framework.BenchmarkTests.csproj @@ -10,6 +10,8 @@ + + @@ -17,4 +19,4 @@ - + \ No newline at end of file diff --git a/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/ContainsOperatorEvalStrategyTests.cs b/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/ContainsOperatorEvalStrategyTests.cs index 1e349728..5ca93263 100644 --- a/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/ContainsOperatorEvalStrategyTests.cs +++ b/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/ContainsOperatorEvalStrategyTests.cs @@ -1,8 +1,8 @@ namespace Rules.Framework.Tests.Evaluation.Interpreted.ValueEvaluation { + using System; using FluentAssertions; using Rules.Framework.Evaluation.Interpreted.ValueEvaluation; - using System; using Xunit; public class ContainsOperatorEvalStrategyTests diff --git a/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/EndsWithOperatorEvalStrategyTests.cs b/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/EndsWithOperatorEvalStrategyTests.cs index 0ad65294..d912365e 100644 --- a/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/EndsWithOperatorEvalStrategyTests.cs +++ b/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/EndsWithOperatorEvalStrategyTests.cs @@ -1,8 +1,8 @@ namespace Rules.Framework.Tests.Evaluation.Interpreted.ValueEvaluation { + using System; using FluentAssertions; using Rules.Framework.Evaluation.Interpreted.ValueEvaluation; - using System; using Xunit; public class EndsWithOperatorEvalStrategyTests diff --git a/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/NotEndsWithOperatorEvalStrategyTests.cs b/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/NotEndsWithOperatorEvalStrategyTests.cs index 4127d85a..e723c9d3 100644 --- a/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/NotEndsWithOperatorEvalStrategyTests.cs +++ b/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/NotEndsWithOperatorEvalStrategyTests.cs @@ -26,18 +26,6 @@ public void Eval_GivenInvalidOperandCombination_ThrowsNotSupportedOperation(obje Assert.Contains(nameof(String), exception.Message); } - [Theory] - [InlineData("Sample text", "random")] - [InlineData("Sample text", "Text")] - public void Eval_GivenLeftOperandNotEndingWithRightOperand_ReturnsTrue(string leftOperand, string rightOperand) - { - // Act - var evaluation = evalStrategy.Eval(leftOperand, rightOperand); - - // Assert - Assert.True(evaluation); - } - [Fact] public void Eval_GivenLeftOperandEndingWithRightOperand_ReturnsFalse() { @@ -51,5 +39,17 @@ public void Eval_GivenLeftOperandEndingWithRightOperand_ReturnsFalse() // Assert Assert.False(evaluation); } + + [Theory] + [InlineData("Sample text", "random")] + [InlineData("Sample text", "Text")] + public void Eval_GivenLeftOperandNotEndingWithRightOperand_ReturnsTrue(string leftOperand, string rightOperand) + { + // Act + var evaluation = evalStrategy.Eval(leftOperand, rightOperand); + + // Assert + Assert.True(evaluation); + } } } \ No newline at end of file diff --git a/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/StartsWithOperatorEvalStrategyTests.cs b/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/StartsWithOperatorEvalStrategyTests.cs index f80d9f36..68e7c3a8 100644 --- a/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/StartsWithOperatorEvalStrategyTests.cs +++ b/tests/Rules.Framework.Tests/Evaluation/Interpreted/ValueEvaluation/StartsWithOperatorEvalStrategyTests.cs @@ -1,8 +1,8 @@ namespace Rules.Framework.Tests.Evaluation.Interpreted.ValueEvaluation { + using System; using FluentAssertions; using Rules.Framework.Evaluation.Interpreted.ValueEvaluation; - using System; using Xunit; public class StartsWithOperatorEvalStrategyTests