feat: Add Data Freshness Check for Self-Healing (#2694)

Co-authored-by: Tom Kerkhove <[email protected]>

Author: hkfgo

.github/workflows/templates-build-push-image.yml

@@ -50,4 +50,4 @@ jobs:
           context: ./src/
           file: ./src/${{ inputs.project_name }}/Dockerfile.linux
           tags: ${{ env.image_commit_uri }},${{ env.image_latest_uri }}
-          push: true
+          push: true

changelog/content/experimental/unreleased.md

@@ -8,6 +8,7 @@ version:
 
 - {{% tag fixed %}} Azure Monitor Scraper: batch based on aggregation in addition to existing criteria
 - {{% tag feature %}} Azure Monitor Scraper: make query lookback range configurable in minutes
+- {{% tag feature %}} Azure Monitor Scraper: add health check based on data freshness
 
 #### Resource Discovery
 

src/Promitor.Agents.Scraper/Configuration/HealthCheckConfiguration.cs

@@ -0,0 +1,13 @@
+namespace Promitor.Agents.Scraper.Configuration
+{
+    public class HealthCheckConfiguration
+    {
+        /// <summary>
+        /// Indicates whether the scraper freshness health check is enabled.
+        /// This health check monitors whether scrapes have been completed recently.
+        /// Defaults to true.
+        /// </summary>
+        public bool EnableScraperFreshnessHealthCheck { get; set; } = true;
+    }
+}
+

src/Promitor.Agents.Scraper/Configuration/ScraperRuntimeConfiguration.cs

@@ -12,5 +12,6 @@ public class ScraperRuntimeConfiguration : RuntimeConfiguration
         public MetricsConfiguration MetricsConfiguration { get; set; } = new();
         public MetricSinkConfiguration MetricSinks { get; set; } = new();
         public ResourceDiscoveryConfiguration ResourceDiscovery { get; set; }
+        public HealthCheckConfiguration HealthCheck { get; set; } = new();
     }
 }

src/Promitor.Agents.Scraper/Docs/Open-Api.xml

@@ -29,5 +29,25 @@ public static IHealthChecksBuilder AddResourceDiscoveryHealthCheck(this IHealthC
 
             return healthChecksBuilder;
         }
+
+        /// <summary>
+        ///     Add health check to validate the scraper has completed a scrape recently
+        /// </summary>
+        /// <param name="healthChecksBuilder">Builder for adding health checks</param>
+        /// <param name="configuration">Configuration of Promitor</param>
+        public static IHealthChecksBuilder AddScraperFreshnessHealthCheck(this IHealthChecksBuilder healthChecksBuilder, IConfiguration configuration)
+        {
+            Guard.NotNull(healthChecksBuilder, nameof(healthChecksBuilder));
+            Guard.NotNull(configuration, nameof(configuration));
+
+            var healthCheckConfiguration = configuration.GetSection("healthCheck").Get<HealthCheckConfiguration>() ?? new HealthCheckConfiguration();
+
+            if (healthCheckConfiguration.EnableScraperFreshnessHealthCheck)
+            {
+                healthChecksBuilder.AddCheck<ScraperResultFreshnessHealthCheck>("Promitor Scraper Data Freshness", HealthStatus.Unhealthy);
+            }
+
+            return healthChecksBuilder;
+        }
     }
 }

src/Promitor.Agents.Scraper/Extensions/IHealthChecksBuilderExtensions.cs

@@ -17,6 +17,7 @@
 using Promitor.Agents.Scraper.Configuration.Sinks;
 using Promitor.Agents.Scraper.Discovery;
 using Promitor.Agents.Scraper.Discovery.Interfaces;
+using Promitor.Agents.Scraper.Health;
 using Promitor.Agents.Scraper.Scheduling;
 using Promitor.Agents.Scraper.Usability;
 using Promitor.Agents.Scraper.Validation.Steps;
@@ -31,6 +32,7 @@
 using Promitor.Core.Scraping.Configuration.Serialization.v1.Core;
 using Promitor.Core.Scraping.Configuration.Serialization.v1.Model;
 using Promitor.Core.Scraping.Factories;
+using Promitor.Agents.Scraper.Runtime;
 using Promitor.Integrations.Azure.Authentication.Configuration;
 using Promitor.Integrations.AzureMonitor.Configuration;
 using Promitor.Integrations.Sinks.Atlassian.Statuspage;
@@ -63,7 +65,7 @@ public static IServiceCollection AddResourceDiscoveryClient(this IServiceCollect
 
             var resourceDiscoveryConfiguration = configuration.Get<ScraperRuntimeConfiguration>();
 
-            if(resourceDiscoveryConfiguration?.ResourceDiscovery?.IsConfigured == true)
+            if (resourceDiscoveryConfiguration?.ResourceDiscovery?.IsConfigured == true)
             {
                 services.AddHttpClient<ResourceDiscoveryClient>(client =>
                 {
@@ -110,12 +112,14 @@ public static IServiceCollection AddAtlassianStatuspageClient(this IServiceColle
         public static IServiceCollection DefineDependencies(this IServiceCollection services)
         {
             Guard.NotNull(services, nameof(services));
-            
+
             services.AddTransient<IMetricsDeclarationProvider, MetricsDeclarationProvider>();
             services.AddTransient<IAzureScrapingSystemMetricsPublisher, AzureScrapingSystemMetricsPublisher>();
             services.AddTransient<MetricScraperFactory>();
             services.AddTransient<ConfigurationSerializer>();
             services.AddSingleton<AzureMonitorClientFactory>();
+            services.AddSingleton<ILastSuccessfulScrapeStore, LastSuccessfulScrapeStore>();
+            services.AddSingleton<IScrapeScheduleProvider, ScrapeScheduleProvider>();
 
             services.AddSingleton<IDeserializer<MetricsDeclarationV1>, V1Deserializer>();
             services.AddSingleton<IDeserializer<AzureMetadataV1>, AzureMetadataDeserializer>();
@@ -245,7 +249,7 @@ private static void AddOpenTelemetryCollectorMetricSink(string collectorUri, str
         }
 
         private static void AddStatsdMetricSink(IServiceCollection services, StatsdSinkConfiguration statsdConfiguration, Table metricSinkAsciiTable)
-        {            
+        {
             metricSinkAsciiTable.AddRow("StatsD", $"Url: {statsdConfiguration.Host}:{statsdConfiguration.Port}.");
             metricSinkAsciiTable.AddRow("", $"Format: {statsdConfiguration.MetricFormat}.");
 
@@ -309,9 +313,9 @@ public static IServiceCollection AddScrapingMutex(this IServiceCollection servic
             }
 
             var serverConfiguration = configuration.GetSection("server").Get<ServerConfiguration>();
-            
+
             services.TryAdd(ServiceDescriptor.Singleton<IScrapingMutex, ScrapingMutex>(_ => ScrapingMutexBuilder(serverConfiguration)));
-            
+
             return services;
         }
 
@@ -333,6 +337,7 @@ public static IServiceCollection ConfigureYamlConfiguration(this IServiceCollect
             services.Configure<TelemetryConfiguration>(configuration.GetSection("telemetry"));
             services.Configure<ServerConfiguration>(configuration.GetSection("server"));
             services.Configure<AuthenticationConfiguration>(configuration.GetSection("authentication"));
+            services.Configure<HealthCheckConfiguration>(configuration.GetSection("healthCheck"));
             services.Configure<PrometheusScrapingEndpointSinkConfiguration>(configuration.GetSection("metricSinks:prometheusScrapingEndpoint"));
             services.Configure<StatsdSinkConfiguration>(configuration.GetSection("metricSinks:statsd"));
             services.Configure<AtlassianStatusPageSinkConfiguration>(configuration.GetSection("metricSinks:atlassianStatuspage"));

src/Promitor.Agents.Scraper/Extensions/IServiceCollectionExtensions.cs

@@ -0,0 +1,16 @@
+using System;
+
+namespace Promitor.Agents.Scraper.Health
+{
+    /// <summary>
+    /// Provides information about the configured scrape schedules
+    /// </summary>
+    public interface IScrapeScheduleProvider
+    {
+        /// <summary>
+        /// Gets the minimum scrape interval across all configured metrics
+        /// </summary>
+        TimeSpan GetMinimumScrapeInterval();
+    }
+}
+

src/Promitor.Agents.Scraper/Health/IScrapeScheduleProvider.cs

@@ -0,0 +1,101 @@
+using System;
+using System.Linq;
+using Cronos;
+using GuardNet;
+using Microsoft.Extensions.Logging;
+using Promitor.Core.Scraping.Configuration.Providers.Interfaces;
+
+namespace Promitor.Agents.Scraper.Health
+{
+    /// <summary>
+    /// Provides information about the configured scrape schedules
+    /// </summary>
+    public class ScrapeScheduleProvider : IScrapeScheduleProvider
+    {
+        private readonly IMetricsDeclarationProvider _metricsDeclarationProvider;
+        private readonly ILogger<ScrapeScheduleProvider> _logger;
+        private TimeSpan? _cachedMinimumInterval;
+
+        public ScrapeScheduleProvider(IMetricsDeclarationProvider metricsDeclarationProvider, ILogger<ScrapeScheduleProvider> logger)
+        {
+            Guard.NotNull(metricsDeclarationProvider, nameof(metricsDeclarationProvider));
+            Guard.NotNull(logger, nameof(logger));
+
+            _metricsDeclarationProvider = metricsDeclarationProvider;
+            _logger = logger;
+        }
+
+        /// <summary>
+        /// Gets the minimum scrape interval across all configured metrics
+        /// </summary>
+        public TimeSpan GetMinimumScrapeInterval()
+        {
+            if (_cachedMinimumInterval.HasValue)
+            {
+                return _cachedMinimumInterval.Value;
+            }
+
+            var metricsDeclaration = _metricsDeclarationProvider.Get(applyDefaults: true);
+            if (metricsDeclaration?.Metrics == null || !metricsDeclaration.Metrics.Any())
+            {
+                _logger.LogWarning("No metrics configured, using default interval of 5 minutes");
+                return TimeSpan.FromMinutes(5);
+            }
+
+            var intervals = metricsDeclaration.Metrics
+                .Select(m => m.Scraping?.Schedule)
+                .Where(schedule => !string.IsNullOrWhiteSpace(schedule))
+                .Distinct()
+                .Select(schedule => CalculateIntervalFromCronExpression(schedule))
+                .Where(interval => interval.HasValue)
+                .Select(interval => interval.Value)
+                .ToList();
+
+            if (!intervals.Any())
+            {
+                _logger.LogWarning("Unable to calculate intervals from cron expressions, using default interval of 5 minutes");
+                return TimeSpan.FromMinutes(5);
+            }
+
+            _cachedMinimumInterval = intervals.Min();
+            _logger.LogInformation("Calculated minimum scrape interval: {MinimumInterval}", _cachedMinimumInterval.Value);
+
+            return _cachedMinimumInterval.Value;
+        }
+
+        private TimeSpan? CalculateIntervalFromCronExpression(string cronExpression)
+        {
+            try
+            {
+                var cron = CronExpression.Parse(cronExpression, CronFormat.IncludeSeconds);
+                var baseTime = DateTimeOffset.UtcNow;
+                var next1 = cron.GetNextOccurrence(baseTime, TimeZoneInfo.Utc);
+
+                if (!next1.HasValue)
+                {
+                    _logger.LogWarning("Unable to calculate next occurrence for cron expression: {CronExpression}", cronExpression);
+                    return null;
+                }
+
+                var next2 = cron.GetNextOccurrence(next1.Value, TimeZoneInfo.Utc);
+
+                if (!next2.HasValue)
+                {
+                    _logger.LogWarning("Unable to calculate second occurrence for cron expression: {CronExpression}", cronExpression);
+                    return null;
+                }
+
+                var interval = next2.Value - next1.Value;
+                _logger.LogDebug("Calculated interval {Interval} for cron expression: {CronExpression}", interval, cronExpression);
+
+                return interval;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "Failed to parse cron expression: {CronExpression}", cronExpression);
+                return null;
+            }
+        }
+    }
+}
+