fix: add aggregation to batch criteria (#2668)

Author: hkfgo

changelog/content/experimental/unreleased.md

@@ -6,7 +6,7 @@ version:
 
 #### Scraper
 
-None.
+- {{% tag fixed %}} Azure Monitor Scraper: batch based on aggregation in addition to existing criteria
 
 #### Resource Discovery
 

src/Promitor.Core.Scraping/Configuration/Model/AzureMetricConfiguration.cs

@@ -67,6 +67,17 @@ public string ToUniqueStringRepresentation()
             }
             sb.Append($"_limit{Limit}");
 
+            if (Aggregation != null)
+            {
+                sb.Append("_agg");
+                sb.Append(Aggregation.Type.ToString());
+
+                if (Aggregation.Interval.HasValue)
+                {
+                    sb.Append("_interval");
+                    sb.Append(Aggregation.Interval.Value.ToString("c"));
+                }
+            }
 
             return sb.ToString();
         }

src/Promitor.Tests.Unit/Core/Metrics/ScrapeDefinitionBatchPropertiesTest.cs

@@ -166,6 +166,25 @@ public void BuildBatchHashKeyDifferentResultDifferentSchedule()
             Assert.NotEqual(hashCode1, hashCode2);
         }
 
+        [Fact]
+        public void BuildBatchHashKeyDifferentResultDifferentAggregation()
+        {
+            var azureMetricConfiguration1 = _mapper.Map<AzureMetricConfiguration>(azureMetricConfigurationBase);
+            azureMetricConfiguration1.Aggregation = new MetricAggregation{Type = PromitorMetricAggregationType.Count};
+            var azureMetricConfiguration2 = _mapper.Map<AzureMetricConfiguration>(azureMetricConfigurationBase);
+            azureMetricConfiguration2.Aggregation = new MetricAggregation{Type = PromitorMetricAggregationType.Average};
+            var logAnalyticsConfiguration = _mapper.Map<LogAnalyticsConfiguration>(logAnalyticsConfigurationBase);
+
+            var scraping = _mapper.Map<Promitor.Core.Scraping.Configuration.Model.Scraping>(scrapingBase);
+
+            var batchProperties = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration1, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId);
+            var batchProperties2 = new ScrapeDefinitionBatchProperties(azureMetricConfiguration: azureMetricConfiguration2, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinition, resourceType: Promitor.Core.Contracts.ResourceType.StorageAccount, scraping: scraping, subscriptionId: subscriptionId);
+
+            var hashCode1 = batchProperties.GetHashCode();
+            var hashCode2 = batchProperties2.GetHashCode();
+            Assert.NotEqual(hashCode1, hashCode2);
+        }
+
         [Fact]
         public void BuildBatchHashKeyTest()
         {

src/Promitor.Tests.Unit/Core/Scraping/Batching/AzureResourceDefinitionBatchingTests.cs

@@ -169,6 +169,30 @@ public void BatchConstructionShouldBeAgnosticToResourceGroup()
             Assert.Single(groupedScrapeDefinitions);
             Assert.Equal(20, groupedScrapeDefinitions[0].ScrapeDefinitions.Count);
         }
+
+        [Fact]
+        public void DifferentAggregationShouldBatchSeparately()
+        {
+            var azureMetricConfigurationMax = _mapper.Map<AzureMetricConfiguration>(azureMetricConfigurationBase);
+            azureMetricConfigurationMax.Aggregation.Type = PromitorMetricAggregationType.Maximum;
+            var azureMetricConfigurationAverage = _mapper.Map<AzureMetricConfiguration>(azureMetricConfigurationBase);
+
+            var scraping = _mapper.Map<Promitor.Core.Scraping.Configuration.Model.Scraping>(scrapingBase);
+            var logAnalyticsConfiguration = _mapper.Map<LogAnalyticsConfiguration>(logAnalyticsConfigurationBase); 
+            var scrapeDefinitions = BuildScrapeDefinitionBatch(
+                azureMetricConfiguration: azureMetricConfigurationMax, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, 
+                resourceType:  ResourceType.StorageAccount, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 10
+            );
+            var differentScrapeDefinitions = BuildScrapeDefinitionBatch(
+                azureMetricConfiguration: azureMetricConfigurationAverage, logAnalyticsConfiguration: logAnalyticsConfiguration, prometheusMetricDefinition: prometheusMetricDefinitionTest, scraping: scraping, 
+                resourceType:  ResourceType.StorageAccount, subscriptionId: subscriptionIdTest, resourceGroupName: resourceGroupNameTest, 10
+            );
+            var groupedScrapeDefinitions = AzureResourceDefinitionBatching.GroupScrapeDefinitions([.. scrapeDefinitions, .. differentScrapeDefinitions], maxBatchSize: batchSize);
+            // expect two batch of 10 each
+            Assert.Equal(2, groupedScrapeDefinitions.Count);
+            Assert.Equal(10, groupedScrapeDefinitions[0].ScrapeDefinitions.Count);
+            Assert.Equal(10, groupedScrapeDefinitions[1].ScrapeDefinitions.Count);
+        }
 
         private static List<ScrapeDefinition<IAzureResourceDefinition>> BuildScrapeDefinitionBatch(
             AzureMetricConfiguration azureMetricConfiguration,
@@ -191,6 +215,7 @@ private static List<ScrapeDefinition<IAzureResourceDefinition>> BuildScrapeDefin
             return batch;
         }
 
+
         private static int CountTotalScrapeDefinitions(List<BatchScrapeDefinition<IAzureResourceDefinition>> groupedScrapeDefinitions) 
         {
             var count = 0;