Report benchmark statistics and comparison

.gitignore

@@ -558,6 +558,7 @@ typings/
 # Specific ones added
 coverage-outputs/**
 .env
+tmp/**
 
 ### Rules Framework Web UI ###
 !src/Rules.Framework.WebUI/node_modules/

run-benchmarks.ps1

@@ -0,0 +1,55 @@
+param ([switch] $KeepBenchmarksFiles)
+
+$originalDir = (Get-Location).Path
+$timestamp = [DateTime]::UtcNow.ToString("yyyyMMdd_hhmmss")
+
+$directoryFound = Get-ChildItem -Path $originalDir -Directory | Select-String -Pattern "tmp"
+if (!$directoryFound) {
+    New-Item -Name "tmp" -ItemType Directory > $null
+}
+
+$directoryFound = Get-ChildItem -Path "$originalDir\\tmp" -Directory | Select-String -Pattern "benchmarks"
+if (!$directoryFound) {
+    New-Item -Name "benchmarks" -ItemType Directory -Path "$originalDir\\tmp" > $null
+}
+
+$directoryFound = Get-ChildItem -Path "$originalDir\\tmp\\benchmarks" -Directory | Select-String -Pattern $timestamp
+if (!$directoryFound) {
+    New-Item -Name $timestamp -ItemType Directory -Path "$originalDir\\tmp\\benchmarks" > $null
+}
+
+$reportDir = "$originalDir\\tmp\\benchmarks\\$timestamp"
+
+# Ensure all packages restored before running benchmarks
+dotnet restore rules-framework.sln
+
+# Build benchmarks binaries
+dotnet build -c Release .\tests\Rules.Framework.BenchmarkTests\Rules.Framework.BenchmarkTests.csproj -o "$reportDir\\bin" --framework net6.0
+
+Set-Location -Path $reportDir
+
+# Run benchmarks
+bin\Rules.Framework.BenchmarkTests.exe -a artifacts
+
+# Determine results file
+$filteredResultsFiles = Get-ChildItem -Path artifacts/results -File -Filter *.md
+if ($filteredResultsFiles) {
+    $resultsFile = $filteredResultsFiles.Name
+
+    # Copy results file
+    Copy-Item -Path artifacts/results/$resultsFile -Destination .
+
+    # Rename file
+    Rename-Item -Path $resultsFile -NewName report.md
+}
+
+if (!$KeepBenchmarksFiles) {
+    if ($directoryFound = Get-ChildItem -Path $reportDir -Directory | Select-String -Pattern "artifacts") {
+        Remove-Item -Path artifacts -Recurse > $null
+    }
+    if ($directoryFound = Get-ChildItem -Path $reportDir -Directory | Select-String -Pattern "bin") {
+        Remove-Item -Path bin -Recurse > $null
+    }
+}
+
+Set-Location -Path $originalDir

src/Rules.Framework/Core/ConditionNodes/ComposedConditionNode.cs

@@ -2,6 +2,7 @@ namespace Rules.Framework.Core.ConditionNodes
 {
     using System;
     using System.Collections.Generic;
+    using System.Collections.ObjectModel;
     using System.Diagnostics;
     using System.Linq;
 

tests/Rules.Framework.BenchmarkTests/CustomBaselineClassifierColumn.cs

@@ -0,0 +1,36 @@
+namespace Rules.Framework.BenchmarkTests
+{
+    using System;
+    using BenchmarkDotNet.Columns;
+    using BenchmarkDotNet.Reports;
+    using BenchmarkDotNet.Running;
+
+    internal class CustomBaselineClassifierColumn : IColumn
+    {
+        private readonly Func<BenchmarkCase, bool> classifyLogicFunc;
+
+        public CustomBaselineClassifierColumn(Func<BenchmarkCase, bool> classifyLogicFunc)
+        {
+            this.classifyLogicFunc = classifyLogicFunc;
+        }
+
+        public bool AlwaysShow => true;
+        public ColumnCategory Category => ColumnCategory.Baseline;
+        public string ColumnName => "Baseline";
+        public string Id => nameof(CustomBaselineClassifierColumn);
+        public bool IsNumeric => false;
+        public string Legend => "Sets wether a test case is a baseline.";
+        public int PriorityInCategory => 0;
+        public UnitType UnitType => UnitType.Dimensionless;
+
+        public string GetValue(Summary summary, BenchmarkCase benchmarkCase)
+            => this.classifyLogicFunc.Invoke(benchmarkCase) ? "Yes" : "No";
+
+        public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
+            => this.GetValue(summary, benchmarkCase);
+
+        public bool IsAvailable(Summary summary) => true;
+
+        public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => false;
+    }
+}

tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkReport.cs

@@ -0,0 +1,17 @@
+namespace Rules.Framework.BenchmarkTests.Exporters.Json
+{
+    using System;
+
+    internal class BenchmarkReport
+    {
+        public DateTime Date { get; set; }
+
+        public Environment? Environment { get; set; }
+
+        public IEnumerable<BenchmarkStatisticsItem>? Statistics { get; set; }
+
+        public IEnumerable<BenchmarkStatisticsComparisonItem>? StatisticsComparison { get; set; }
+
+        public string Title { get; set; } = string.Empty;
+    }
+}

tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsComparisonItem.cs

@@ -0,0 +1,23 @@
+namespace Rules.Framework.BenchmarkTests.Exporters.Json
+{
+    internal class BenchmarkStatisticsComparisonItem
+    {
+        public BenchmarkStatisticsValue? AllocatedMemoryRate { get; set; }
+
+        public BenchmarkStatisticsValue? BaselineAllocatedMemory { get; set; }
+
+        public BenchmarkStatisticsValue? BaselineMeanTimeTaken { get; set; }
+
+        public string BaselineParameters { get; set; } = string.Empty;
+
+        public BenchmarkStatisticsValue? CompareAllocatedMemory { get; set; }
+
+        public BenchmarkStatisticsValue? CompareMeanTimeTaken { get; set; }
+
+        public string CompareParameters { get; set; } = string.Empty;
+
+        public string Key { get; set; } = string.Empty;
+
+        public BenchmarkStatisticsValue? MeanTimeTakenCompareRate { get; set; }
+    }
+}

tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsItem.cs

@@ -0,0 +1,23 @@
+namespace Rules.Framework.BenchmarkTests.Exporters.Json
+{
+    internal class BenchmarkStatisticsItem
+    {
+        public BenchmarkStatisticsValue? AllocatedMemory { get; set; }
+
+        public string Baseline { get; set; } = string.Empty;
+
+        public BenchmarkStatisticsValue? BranchInstructionsPerOp { get; set; }
+
+        public BenchmarkStatisticsValue? BranchMispredictionsPerOp { get; set; }
+
+        public BenchmarkStatisticsValue? Gen0Collects { get; set; }
+
+        public string Key { get; set; } = string.Empty;
+
+        public BenchmarkStatisticsValue? MeanTimeTaken { get; set; }
+
+        public string Parameters { get; set; } = string.Empty;
+
+        public BenchmarkStatisticsValue? StandardError { get; set; }
+    }
+}

tests/Rules.Framework.BenchmarkTests/Exporters/Json/BenchmarkStatisticsValue.cs

@@ -0,0 +1,11 @@
+namespace Rules.Framework.BenchmarkTests.Exporters.Json
+{
+    internal class BenchmarkStatisticsValue
+    {
+        public string Format { get; set; } = string.Empty;
+
+        public string Unit { get; set; } = string.Empty;
+
+        public decimal Value { get; set; }
+    }
+}

tests/Rules.Framework.BenchmarkTests/Exporters/Json/CustomJsonExporter.cs

@@ -0,0 +1,167 @@
+namespace Rules.Framework.BenchmarkTests.Exporters.Json
+{
+    using System.Collections.Generic;
+    using BenchmarkDotNet.Exporters;
+    using BenchmarkDotNet.Loggers;
+    using BenchmarkDotNet.Reports;
+    using Newtonsoft.Json;
+
+    internal class CustomJsonExporter : ExporterBase
+    {
+        public CustomJsonExporter(bool indentJson)
+        {
+            this.IndentJson = indentJson;
+        }
+
+        public static IExporter Compressed => new CustomJsonExporter(indentJson: false);
+
+        public static IExporter Default => Indented;
+
+        public static IExporter Indented => new CustomJsonExporter(indentJson: true);
+
+        protected override string FileExtension => "json";
+
+        private bool IndentJson { get; }
+
+        public override void ExportToLog(Summary summary, ILogger logger)
+        {
+            var report = CustomJsonExporter.CreateReport(summary);
+
+            var settings = new JsonSerializerSettings
+            {
+                Formatting = Formatting.None
+            };
+
+            if (this.IndentJson)
+            {
+                settings.Formatting = Formatting.Indented;
+            }
+
+            string jsonText = JsonConvert.SerializeObject(report, settings);
+
+            logger.WriteLine(jsonText);
+        }
+
+        private static decimal CalculateRate(decimal baselineValue, decimal compareValue)
+            => ((baselineValue - compareValue) / baselineValue) * 100;
+
+        private static BenchmarkStatisticsComparisonItem CreateBenchmarkStatisticsComparisonItem(BenchmarkStatisticsItem? baselineStatisticsItem, BenchmarkStatisticsItem? nonBaselineStatisticsItem) => new BenchmarkStatisticsComparisonItem
+        {
+            AllocatedMemoryRate = new BenchmarkStatisticsValue
+            {
+                Format = "0.##",
+                Unit = "%",
+                Value = CalculateRate(
+                                        baselineStatisticsItem.AllocatedMemory?.Value ?? 0,
+                                        nonBaselineStatisticsItem.AllocatedMemory?.Value ?? 0),
+            },
+            BaselineAllocatedMemory = baselineStatisticsItem.AllocatedMemory,
+            BaselineMeanTimeTaken = baselineStatisticsItem.MeanTimeTaken,
+            BaselineParameters = baselineStatisticsItem.Parameters,
+            CompareAllocatedMemory = nonBaselineStatisticsItem.AllocatedMemory,
+            CompareMeanTimeTaken = nonBaselineStatisticsItem.MeanTimeTaken,
+            CompareParameters = nonBaselineStatisticsItem.Parameters,
+            Key = baselineStatisticsItem.Key,
+            MeanTimeTakenCompareRate = new BenchmarkStatisticsValue
+            {
+                Format = "0.##",
+                Unit = "%",
+                Value = CalculateRate(
+                                        baselineStatisticsItem.MeanTimeTaken?.Value ?? 0,
+                                        nonBaselineStatisticsItem.MeanTimeTaken?.Value ?? 0),
+            }
+        };
+
+        private static BenchmarkStatisticsItem CreateBenchmarkStatisticsItem(SummaryTable.SummaryTableColumn baselineClassifierColumn, ref int current, BenchmarkDotNet.Reports.BenchmarkReport benchmarkReport)
+        {
+            var allocatedMemoryMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "Allocated Memory");
+            var branchInstructionsMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "BranchInstructions");
+            var branchMispredictionsMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "BranchMispredictions");
+            var gen0CollectsMetric = benchmarkReport.Metrics.FirstOrDefault(m => m.Key == "Gen0Collects");
+
+            var statisticsItem = new BenchmarkStatisticsItem
+            {
+                AllocatedMemory = allocatedMemoryMetric.Key is null ? null : CreateBenchmarkStatisticsValue(allocatedMemoryMetric),
+                Baseline = baselineClassifierColumn.Content[current++],
+                BranchInstructionsPerOp =
+                    branchInstructionsMetric.Key is null ? null : CreateBenchmarkStatisticsValue(branchInstructionsMetric),
+                BranchMispredictionsPerOp =
+                    branchMispredictionsMetric.Key is null ? null : CreateBenchmarkStatisticsValue(branchMispredictionsMetric),
+                Gen0Collects = gen0CollectsMetric.Key is null ? null : CreateBenchmarkStatisticsValue(gen0CollectsMetric),
+                Key = $"{benchmarkReport.BenchmarkCase.Descriptor.Type.Name}.{benchmarkReport.BenchmarkCase.Descriptor.WorkloadMethod.Name}",
+                MeanTimeTaken = new BenchmarkStatisticsValue
+                {
+                    Format = "N0",
+                    Unit = "ns",
+                    Value = Convert.ToDecimal(benchmarkReport.ResultStatistics?.Mean ?? 0),
+                },
+                Parameters = benchmarkReport.BenchmarkCase.Parameters.DisplayInfo,
+                StandardError = new BenchmarkStatisticsValue
+                {
+                    Format = "N0",
+                    Unit = "ns",
+                    Value = Convert.ToDecimal(benchmarkReport.ResultStatistics?.StandardError ?? 0),
+                },
+            };
+            return statisticsItem;
+        }
+
+        private static BenchmarkStatisticsValue CreateBenchmarkStatisticsValue(KeyValuePair<string, Metric> allocatedMemoryMetric) => new BenchmarkStatisticsValue
+        {
+            Format = allocatedMemoryMetric.Value.Descriptor.NumberFormat,
+            Unit = allocatedMemoryMetric.Value.Descriptor.Unit,
+            Value = Convert.ToDecimal(allocatedMemoryMetric.Value.Value),
+        };
+
+        private static Environment CreateEnvironment(Summary summary) => new Environment
+        {
+            Architecture = summary.HostEnvironmentInfo.Architecture,
+            BenchmarkDotNetCaption = "BenchmarkDotNet",
+            BenchmarkDotNetVersion = summary.HostEnvironmentInfo.BenchmarkDotNetVersion,
+            BuildConfiguration = summary.HostEnvironmentInfo.Configuration,
+            DotNetCliVersion = summary.HostEnvironmentInfo.DotNetSdkVersion.Value,
+            DotNetRuntimeVersion = summary.HostEnvironmentInfo.RuntimeVersion,
+            LogicalCoreCount = summary.HostEnvironmentInfo.CpuInfo.Value.LogicalCoreCount.GetValueOrDefault(0),
+            PhysicalCoreCount = summary.HostEnvironmentInfo.CpuInfo.Value.PhysicalCoreCount.GetValueOrDefault(0),
+            ProcessorName = summary.HostEnvironmentInfo.CpuInfo.Value.ProcessorName,
+        };
+
+        private static BenchmarkReport CreateReport(Summary summary)
+        {
+            var report = new BenchmarkReport
+            {
+                Date = DateTime.UtcNow,
+                Environment = CreateEnvironment(summary),
+                Title = summary.Title,
+            };
+            var statistics = new List<BenchmarkStatisticsItem>(summary.Reports.Length);
+
+            var baselineClassifierColumn = summary.Table.Columns.First(c => c.OriginalColumn.ColumnName == "Baseline");
+            var current = 0;
+            foreach (var benchmarkReport in summary.Reports)
+            {
+                var statisticsItem = CreateBenchmarkStatisticsItem(baselineClassifierColumn, ref current, benchmarkReport);
+
+                statistics.Add(statisticsItem);
+            }
+            report.Statistics = statistics;
+
+            var baselineStatisticsItems = statistics.Where(i => string.Equals(i.Baseline, "Yes"));
+            var nonBaselineStatisticsItems = statistics.Where(i => string.Equals(i.Baseline, "No"));
+            var statisticsComparison = new List<BenchmarkStatisticsComparisonItem>();
+
+            foreach (var baselineStatisticsItem in baselineStatisticsItems)
+            {
+                foreach (var nonBaselineStatisticsItem in nonBaselineStatisticsItems.Where(i => string.Equals(i.Key, baselineStatisticsItem.Key)))
+                {
+                    var statisticsComparisonItem = CreateBenchmarkStatisticsComparisonItem(baselineStatisticsItem, nonBaselineStatisticsItem);
+
+                    statisticsComparison.Add(statisticsComparisonItem);
+                }
+            }
+            report.StatisticsComparison = statisticsComparison;
+
+            return report;
+        }
+    }
+}

tests/Rules.Framework.BenchmarkTests/Exporters/Json/Environment.cs

@@ -0,0 +1,23 @@
+namespace Rules.Framework.BenchmarkTests.Exporters.Json
+{
+    internal class Environment
+    {
+        public string Architecture { get; set; } = string.Empty;
+
+        public string BenchmarkDotNetCaption { get; set; } = string.Empty;
+
+        public string BenchmarkDotNetVersion { get; set; } = string.Empty;
+
+        public string BuildConfiguration { get; set; } = string.Empty;
+
+        public string DotNetCliVersion { get; set; } = string.Empty;
+
+        public string DotNetRuntimeVersion { get; set; } = string.Empty;
+
+        public int LogicalCoreCount { get; set; }
+
+        public int PhysicalCoreCount { get; set; }
+
+        public string ProcessorName { get; set; } = string.Empty;
+    }
+}