fix: correctly log job timeout (#2629)

Author: hkfgo

changelog/content/experimental/unreleased.md

@@ -7,6 +7,7 @@ version:
 #### Scraper
 
 None.
+
 #### Resource Discovery
 
 None.

src/Promitor.Agents.Core/Configuration/Defaults.cs

@@ -24,7 +24,7 @@ public static class Concurrency
             /// On rare occasions, scrape jobs can hang and subsequent jobs cannot run, as they cannot acquire a mutex. 
             ///  MutexTimeoutSeconds setting ensures scraping mutex is released.  
             /// </summary>
-            public static int MutexTimeoutSeconds { get; } = 30;
+            public static int MutexTimeoutSeconds { get; } = 90;
         }
 
         public class Telemetry

src/Promitor.Agents.Scraper/Scheduling/ResourcesScrapingJob.cs

@@ -142,14 +142,11 @@ public async Task ExecuteAsync(CancellationToken cancellationToken)
             {
                 var mutexReleasedAfterSeconds = _concurrencyConfiguration.Value.MutexTimeoutSeconds;
                 var timeoutCancellationTokenSource = new CancellationTokenSource();
+                Logger.LogInformation("Timeout after {Timeout}", mutexReleasedAfterSeconds);
+
                 timeoutCancellationTokenSource.CancelAfter(mutexReleasedAfterSeconds * 1000);
-                timeoutCancellationTokenSource.Token.Register(() => {
-                    Logger.LogError("Scrape job {JobName} was cancelled due to timeout. However, dangling async tasks " +
-                                    "may be running for an unbounded amount of time. In the rare case where " +
-                                    "many such timeouts occur, consider restarting the Scraper Agent.", Name);     
-                });
 
-                Logger.LogWarning("Init timeout token");
+                Logger.LogInformation("Init timeout token");
                 // to enforce timeout in addition to cancellationToken passed down by .NET 
                 var composedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationTokenSource.Token);
                 var scrapeDefinitions = await GetAllScrapeDefinitions(composedCancellationTokenSource.Token);
@@ -377,7 +374,14 @@ private async Task WorkWrapper(Func<Task> work, CancellationToken cancellationTo
                 cancellationToken.Register(() => {
                     tcs.TrySetResult(null);
                 });
-                await Task.WhenAny(work(), tcs.Task);
+                var completedTask = await Task.WhenAny(work(), tcs.Task);
+                if (completedTask == tcs.Task)
+                {
+                    Logger.LogError("Scrape job {JobName} was cancelled due to timeout. However, dangling async tasks " +
+                          "may be running for an unbounded amount of time. In the rare case where " +
+                          "many such timeouts occur, consider restarting the Scraper Agent or tuning the mutex timeout configuration.", Name);
+                    throw new OperationCanceledException(cancellationToken);
+                }
             }
             finally
             {