diff --git a/src/Config/Azure.DataApiBuilder.Config.csproj b/src/Config/Azure.DataApiBuilder.Config.csproj index 25dd0716f9..a494bc38ae 100644 --- a/src/Config/Azure.DataApiBuilder.Config.csproj +++ b/src/Config/Azure.DataApiBuilder.Config.csproj @@ -18,6 +18,7 @@ + @@ -25,7 +26,7 @@ - + diff --git a/src/Config/Converters/FileSinkConverter.cs b/src/Config/Converters/FileSinkConverter.cs index e5d68ca20e..e0107a11f6 100644 --- a/src/Config/Converters/FileSinkConverter.cs +++ b/src/Config/Converters/FileSinkConverter.cs @@ -4,6 +4,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Azure.DataApiBuilder.Config.ObjectModel; +using Serilog; namespace Azure.DataApiBuilder.Config.Converters; class FileSinkConverter : JsonConverter @@ -31,7 +32,7 @@ public FileSinkConverter(bool replaceEnvVar) { bool? enabled = null; string? path = null; - RollingIntervalMode? rollingInterval = null; + RollingInterval? rollingInterval = null; int? retainedFileCountLimit = null; int? fileSizeLimitBytes = null; @@ -66,7 +67,7 @@ public FileSinkConverter(bool replaceEnvVar) case "rolling-interval": if (reader.TokenType is not JsonTokenType.Null) { - rollingInterval = EnumExtensions.Deserialize(reader.DeserializeString(_replaceEnvVar)!); + rollingInterval = EnumExtensions.Deserialize(reader.DeserializeString(_replaceEnvVar)!); } break; diff --git a/src/Config/ObjectModel/FileSinkOptions.cs b/src/Config/ObjectModel/FileSinkOptions.cs index e6cd20810b..7e6674fcad 100644 --- a/src/Config/ObjectModel/FileSinkOptions.cs +++ b/src/Config/ObjectModel/FileSinkOptions.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using System.Text.Json.Serialization; +using Serilog; namespace Azure.DataApiBuilder.Config.ObjectModel; @@ -19,12 +20,12 @@ public record FileSinkOptions /// /// Default path for File Sink. /// - public const string DEFAULT_PATH = "/logs/dab-log.txt"; + public const string DEFAULT_PATH = @"logs\dab-log.txt"; /// /// Default rolling interval for File Sink. /// - public const string DEFAULT_ROLLING_INTERVAL = nameof(RollingIntervalMode.Day); + public const string DEFAULT_ROLLING_INTERVAL = nameof(Serilog.RollingInterval.Day); /// /// Default retained file count limit for File Sink. @@ -44,25 +45,25 @@ public record FileSinkOptions /// /// Path to the file where logs will be uploaded. /// - public string? Path { get; init; } + public string Path { get; init; } /// /// Time it takes for files with logs to be discarded. /// - public string? RollingInterval { get; init; } + public string RollingInterval { get; init; } /// /// Amount of files that can exist simultaneously in which logs are saved. /// - public int? RetainedFileCountLimit { get; init; } + public int RetainedFileCountLimit { get; init; } /// /// File size limit in bytes before a new file needs to be created. /// - public int? FileSizeLimitBytes { get; init; } + public int FileSizeLimitBytes { get; init; } [JsonConstructor] - public FileSinkOptions(bool? enabled = null, string? path = null, RollingIntervalMode? rollingInterval = null, int? retainedFileCountLimit = null, int? fileSizeLimitBytes = null) + public FileSinkOptions(bool? enabled = null, string? path = null, RollingInterval? rollingInterval = null, int? retainedFileCountLimit = null, int? fileSizeLimitBytes = null) { if (enabled is not null) { @@ -86,7 +87,7 @@ public FileSinkOptions(bool? enabled = null, string? path = null, RollingInterva if (rollingInterval is not null) { - RollingInterval = rollingInterval.ToString(); + RollingInterval = ((RollingInterval)rollingInterval).ToString(); UserProvidedRollingInterval = true; } else @@ -96,7 +97,7 @@ public FileSinkOptions(bool? enabled = null, string? path = null, RollingInterva if (retainedFileCountLimit is not null) { - RetainedFileCountLimit = retainedFileCountLimit; + RetainedFileCountLimit = (int)retainedFileCountLimit; UserProvidedRetainedFileCountLimit = true; } else @@ -106,7 +107,7 @@ public FileSinkOptions(bool? enabled = null, string? path = null, RollingInterva if (fileSizeLimitBytes is not null) { - FileSizeLimitBytes = fileSizeLimitBytes; + FileSizeLimitBytes = (int)fileSizeLimitBytes; UserProvidedFileSizeLimitBytes = true; } else diff --git a/src/Config/ObjectModel/RollingIntervalMode.cs b/src/Config/ObjectModel/RollingIntervalMode.cs deleted file mode 100644 index df6d77e67b..0000000000 --- a/src/Config/ObjectModel/RollingIntervalMode.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -using System.Text.Json.Serialization; - -namespace Azure.DataApiBuilder.Config.ObjectModel; - -/// -/// Represents the rolling interval options for file sink. -/// The time it takes between the creation of new files. -/// -[JsonConverter(typeof(JsonStringEnumConverter))] -public enum RollingIntervalMode -{ - /// - /// The log file will never roll; no time period information will be appended to the log file name. - /// - Infinite, - - /// - /// Roll every year. Filenames will have a four-digit year appended in the pattern yyyy. - /// - Year, - - /// - /// Roll every calendar month. Filenames will have yyyyMM appended. - /// - Month, - - /// - /// Roll every day. Filenames will have yyyyMMdd appended. - /// - Day, - - /// - /// Roll every hour. Filenames will have yyyyMMddHH appended. - /// - Hour, - - /// - /// Roll every minute. Filenames will have yyyyMMddHHmm appended. - /// - Minute -} diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index d00f43c478..da600c9f63 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -60,6 +60,8 @@ + + diff --git a/src/Service.Tests/Configuration/ConfigurationTests.cs b/src/Service.Tests/Configuration/ConfigurationTests.cs index 98cb89d919..2522806049 100644 --- a/src/Service.Tests/Configuration/ConfigurationTests.cs +++ b/src/Service.Tests/Configuration/ConfigurationTests.cs @@ -47,6 +47,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Moq.Protected; +using Serilog; using VerifyMSTest; using static Azure.DataApiBuilder.Config.FileSystemRuntimeConfigLoader; using static Azure.DataApiBuilder.Service.Tests.Configuration.ConfigurationEndpoints; @@ -4170,21 +4171,21 @@ public void AzureLogAnalyticsSerialization( /// [DataTestMethod] [TestCategory(TestCategory.MSSQL)] - [DataRow(true, "/file/path/exists.txt", RollingIntervalMode.Minute, 27, 256, true, "/file/path/exists.txt", RollingIntervalMode.Minute, 27, 256)] - [DataRow(true, "/test/path.csv", RollingIntervalMode.Hour, 10, 3000, true, "/test/path.csv", RollingIntervalMode.Hour, 10, 3000)] - [DataRow(false, "C://absolute/file/path.log", RollingIntervalMode.Month, 2147483647, 2048, false, "C://absolute/file/path.log", RollingIntervalMode.Month, 2147483647, 2048)] - [DataRow(false, "D://absolute/test/path.txt", RollingIntervalMode.Year, 10, 2147483647, false, "D://absolute/test/path.txt", RollingIntervalMode.Year, 10, 2147483647)] - [DataRow(false, "", RollingIntervalMode.Infinite, 5, 512, false, "", RollingIntervalMode.Infinite, 5, 512)] - [DataRow(null, null, null, null, null, false, "/logs/dab-log.txt", RollingIntervalMode.Day, 1, 1048576)] + [DataRow(true, "/file/path/exists.txt", RollingInterval.Minute, 27, 256, true, "/file/path/exists.txt", RollingInterval.Minute, 27, 256)] + [DataRow(true, "/test/path.csv", RollingInterval.Hour, 10, 3000, true, "/test/path.csv", RollingInterval.Hour, 10, 3000)] + [DataRow(false, "C://absolute/file/path.log", RollingInterval.Month, 2147483647, 2048, false, "C://absolute/file/path.log", RollingInterval.Month, 2147483647, 2048)] + [DataRow(false, "D://absolute/test/path.txt", RollingInterval.Year, 10, 2147483647, false, "D://absolute/test/path.txt", RollingInterval.Year, 10, 2147483647)] + [DataRow(false, "", RollingInterval.Infinite, 5, 512, false, "", RollingInterval.Infinite, 5, 512)] + [DataRow(null, null, null, null, null, false, "/logs/dab-log.txt", RollingInterval.Day, 1, 1048576)] public void FileSinkSerialization( bool? enabled, string? path, - RollingIntervalMode? rollingInterval, + RollingInterval? rollingInterval, int? retainedFileCountLimit, int? fileSizeLimitBytes, bool expectedEnabled, string expectedPath, - RollingIntervalMode expectedRollingInterval, + RollingInterval expectedRollingInterval, int expectedRetainedFileCountLimit, int expectedFileSizeLimitBytes) { diff --git a/src/Service.Tests/Configuration/Telemetry/FileSinkTests.cs b/src/Service.Tests/Configuration/Telemetry/FileSinkTests.cs new file mode 100644 index 0000000000..077e0098d8 --- /dev/null +++ b/src/Service.Tests/Configuration/Telemetry/FileSinkTests.cs @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using Azure.DataApiBuilder.Config.ObjectModel; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Serilog; +using Serilog.Core; +using static Azure.DataApiBuilder.Service.Tests.Configuration.ConfigurationTests; + +namespace Azure.DataApiBuilder.Service.Tests.Configuration.Telemetry; + +/// +/// Contains tests for File Sink functionality. +/// +[TestClass, TestCategory(TestCategory.MSSQL)] +public class FileSinkTests +{ + public TestContext TestContext { get; set; } + + private const string CONFIG_WITH_TELEMETRY = "dab-file-sink-test-config.json"; + private const string CONFIG_WITHOUT_TELEMETRY = "dab-no-file-sink-test-config.json"; + private static RuntimeConfig _configuration; + + /// + /// This is a helper function that creates runtime config file with specified telemetry options. + /// + /// Name of the config file to be created. + /// Whether File Sink is enabled or not. + /// Path where logs will be sent to. + /// Time it takes for logs to roll over to next file. + private static void SetUpTelemetryInConfig(string configFileName, bool isFileSinkEnabled, string fileSinkPath, RollingInterval? rollingInterval = null) + { + DataSource dataSource = new(DatabaseType.MSSQL, + GetConnectionStringFromEnvironmentConfig(environment: TestCategory.MSSQL), Options: null); + + _configuration = InitMinimalRuntimeConfig(dataSource, graphqlOptions: new(), restOptions: new()); + + TelemetryOptions _testTelemetryOptions = new(File: new FileSinkOptions(isFileSinkEnabled, fileSinkPath, rollingInterval)); + _configuration = _configuration with { Runtime = _configuration.Runtime with { Telemetry = _testTelemetryOptions } }; + + File.WriteAllText(configFileName, _configuration.ToJson()); + } + + /// + /// Cleans up the test environment by deleting the runtime config with telemetry options. + /// + [TestCleanup] + public void CleanUpTelemetryConfig() + { + if (File.Exists(CONFIG_WITH_TELEMETRY)) + { + File.Delete(CONFIG_WITH_TELEMETRY); + } + + if (File.Exists(CONFIG_WITHOUT_TELEMETRY)) + { + File.Delete(CONFIG_WITHOUT_TELEMETRY); + } + } + + /// + /// Tests if the services are correctly enabled for File Sink. + /// + [TestMethod] + public void TestFileSinkServicesEnabled() + { + // Arrange + SetUpTelemetryInConfig(CONFIG_WITH_TELEMETRY, true, "/dab-log-test/file-sink-file.txt"); + + string[] args = new[] + { + $"--ConfigFileName={CONFIG_WITH_TELEMETRY}" + }; + using TestServer server = new(Program.CreateWebHostBuilder(args)); + + // Additional assertions to check if File Sink is enabled correctly in services + IServiceProvider serviceProvider = server.Services; + LoggerConfiguration serilogLoggerConfiguration = serviceProvider.GetService(); + Logger serilogLogger = serviceProvider.GetService(); + + // If serilogLoggerConfiguration and serilogLogger are not null, File Sink is enabled + Assert.IsNotNull(serilogLoggerConfiguration, "LoggerConfiguration for Serilog should be registered."); + Assert.IsNotNull(serilogLogger, "Logger for Serilog should be registered."); + } + + /// + /// Tests if the logs are flushed to the proper path when File Sink is enabled. + /// + /// + /// Tests if the logs are flushed to the proper path when File Sink is enabled. + /// + [DataTestMethod] + [DataRow("file-sink-test-file.txt")] + [DataRow("file-sink-test-file.log")] + [DataRow("file-sink-test-file.csv")] + public async Task TestFileSinkSucceed(string fileName) + { + // Arrange + SetUpTelemetryInConfig(CONFIG_WITH_TELEMETRY, true, fileName, RollingInterval.Infinite); + + string[] args = new[] + { + $"--ConfigFileName={CONFIG_WITH_TELEMETRY}" + }; + using TestServer server = new(Program.CreateWebHostBuilder(args)); + + // Act + using (HttpClient client = server.CreateClient()) + { + HttpRequestMessage restRequest = new(HttpMethod.Get, "/api/Book"); + await client.SendAsync(restRequest); + } + + server.Dispose(); + + // Assert + Assert.IsTrue(File.Exists(fileName)); + + bool containsInfo = false; + string[] allLines = File.ReadAllLines(fileName); + foreach (string line in allLines) + { + containsInfo = line.Contains("INF"); + if (containsInfo) + { + break; + } + } + + Assert.IsTrue(containsInfo); + } + + /// + /// Tests if the services are correctly disabled for File Sink. + /// + [TestMethod] + public void TestFileSinkServicesDisabled() + { + // Arrange + SetUpTelemetryInConfig(CONFIG_WITHOUT_TELEMETRY, false, null); + + string[] args = new[] + { + $"--ConfigFileName={CONFIG_WITHOUT_TELEMETRY}" + }; + using TestServer server = new(Program.CreateWebHostBuilder(args)); + + // Additional assertions to check if File Sink is enabled correctly in services + IServiceProvider serviceProvider = server.Services; + LoggerConfiguration serilogLoggerConfiguration = serviceProvider.GetService(); + Logger serilogLogger = serviceProvider.GetService(); + + // If serilogLoggerConfiguration and serilogLogger are null, File Sink is disabled + Assert.IsNull(serilogLoggerConfiguration, "LoggerConfiguration for Serilog should not be registered."); + Assert.IsNull(serilogLogger, "Logger for Serilog should not be registered."); + } +} diff --git a/src/Service/Azure.DataApiBuilder.Service.csproj b/src/Service/Azure.DataApiBuilder.Service.csproj index bb21361d4b..9f1558e504 100644 --- a/src/Service/Azure.DataApiBuilder.Service.csproj +++ b/src/Service/Azure.DataApiBuilder.Service.csproj @@ -75,6 +75,8 @@ + + diff --git a/src/Service/Program.cs b/src/Service/Program.cs index 7009e489ce..1059fd52ff 100644 --- a/src/Service/Program.cs +++ b/src/Service/Program.cs @@ -12,6 +12,7 @@ using Azure.DataApiBuilder.Service.Telemetry; using Microsoft.ApplicationInsights; using Microsoft.AspNetCore; +using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; @@ -20,6 +21,9 @@ using OpenTelemetry.Exporter; using OpenTelemetry.Logs; using OpenTelemetry.Resources; +using Serilog; +using Serilog.Core; +using Serilog.Extensions.Logging; namespace Azure.DataApiBuilder.Service { @@ -132,9 +136,11 @@ private static ParseResult GetParseResult(Command cmd, string[] args) /// /// Creates a LoggerFactory and add filter with the given LogLevel. /// - /// minimum log level. + /// Minimum log level. /// Telemetry client - public static ILoggerFactory GetLoggerFactoryForLogLevel(LogLevel logLevel, TelemetryClient? appTelemetryClient = null, LogLevelInitializer? logLevelInitializer = null) + /// Hot-reloadable log level + /// Core Serilog logging pipeline + public static ILoggerFactory GetLoggerFactoryForLogLevel(LogLevel logLevel, TelemetryClient? appTelemetryClient = null, LogLevelInitializer? logLevelInitializer = null, Logger? serilogLogger = null) { return LoggerFactory .Create(builder => @@ -209,6 +215,20 @@ public static ILoggerFactory GetLoggerFactoryForLogLevel(LogLevel logLevel, Tele } } + if (Startup.FileSinkOptions.Enabled && serilogLogger is not null) + { + builder.AddSerilog(serilogLogger); + + if (logLevelInitializer is null) + { + builder.AddFilter(category: string.Empty, logLevel); + } + else + { + builder.AddFilter(category: string.Empty, level => level >= logLevelInitializer.MinLogLevel); + } + } + builder.AddConsole(); }); } diff --git a/src/Service/Startup.cs b/src/Service/Startup.cs index a98f2ab15c..ce6b3077a4 100644 --- a/src/Service/Startup.cs +++ b/src/Service/Startup.cs @@ -58,6 +58,8 @@ using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; +using Serilog; +using Serilog.Core; using StackExchange.Redis; using ZiggyCreatures.Caching.Fusion; using ZiggyCreatures.Caching.Fusion.Backplane.StackExchangeRedis; @@ -76,6 +78,7 @@ public class Startup(IConfiguration configuration, ILogger logger) public static ApplicationInsightsOptions AppInsightsOptions = new(); public static OpenTelemetryOptions OpenTelemetryOptions = new(); public static AzureLogAnalyticsOptions AzureLogAnalyticsOptions = new(); + public static FileSinkOptions FileSinkOptions = new(); public const string NO_HTTPS_REDIRECT_FLAG = "--no-https-redirect"; private readonly HotReloadEventHandler _hotReloadEventHandler = new(); private RuntimeConfigProvider? _configProvider; @@ -192,6 +195,23 @@ public void ConfigureServices(IServiceCollection services) services.AddHostedService(sp => sp.GetRequiredService()); } + if (runtimeConfigAvailable + && runtimeConfig?.Runtime?.Telemetry?.File is not null + && runtimeConfig.Runtime.Telemetry.File.Enabled) + { + services.AddSingleton(sp => + { + FileSinkOptions options = runtimeConfig.Runtime.Telemetry.File; + return new LoggerConfiguration().WriteTo.File( + path: options.Path, + rollingInterval: (RollingInterval)Enum.Parse(typeof(RollingInterval), options.RollingInterval), + retainedFileCountLimit: options.RetainedFileCountLimit, + fileSizeLimitBytes: options.FileSizeLimitBytes, + rollOnFileSizeLimit: true); + }); + services.AddSingleton(sp => sp.GetRequiredService().MinimumLevel.Verbose().CreateLogger()); + } + services.AddSingleton(implementationFactory: serviceProvider => { LogLevelInitializer logLevelInit = new(MinimumLogLevel, typeof(RuntimeConfigValidator).FullName, _configProvider, _hotReloadEventHandler); @@ -538,6 +558,7 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env, RuntimeC ConfigureApplicationInsightsTelemetry(app, runtimeConfig); ConfigureOpenTelemetry(runtimeConfig); ConfigureAzureLogAnalytics(runtimeConfig); + ConfigureFileSink(app, runtimeConfig); // Config provided before starting the engine. isRuntimeReady = PerformOnConfigChangeAsync(app).Result; @@ -709,8 +730,9 @@ public static ILoggerFactory CreateLoggerFactoryForHostedAndNonHostedScenario(IS } TelemetryClient? appTelemetryClient = serviceProvider.GetService(); + Logger? serilogLogger = serviceProvider.GetService(); - return Program.GetLoggerFactoryForLogLevel(logLevelInitializer.MinLogLevel, appTelemetryClient, logLevelInitializer); + return Program.GetLoggerFactoryForLogLevel(logLevelInitializer.MinLogLevel, appTelemetryClient, logLevelInitializer, serilogLogger); } /// @@ -941,6 +963,44 @@ private void ConfigureAzureLogAnalytics(RuntimeConfig runtimeConfig) } } + /// + /// Configure File Sink based on the loaded runtime configuration. If File Sink + /// is enabled, we can track different events and metrics. + /// + /// The application builder. + /// The provider used to load runtime configuration. + private void ConfigureFileSink(IApplicationBuilder app, RuntimeConfig runtimeConfig) + { + if (runtimeConfig?.Runtime?.Telemetry is not null + && runtimeConfig.Runtime.Telemetry.File is not null) + { + FileSinkOptions = runtimeConfig.Runtime.Telemetry.File; + + if (!FileSinkOptions.Enabled) + { + _logger.LogInformation("File is disabled."); + return; + } + + if (string.IsNullOrWhiteSpace(FileSinkOptions.Path)) + { + _logger.LogError("Logs won't be sent to File because the Path is not available in the config file."); + return; + } + + Logger? serilogLogger = app.ApplicationServices.GetService(); + if (serilogLogger is null) + { + _logger.LogError("Serilog Logger Configuration is not set."); + return; + } + + // Updating Startup Logger to Log from Startup Class. + ILoggerFactory? loggerFactory = Program.GetLoggerFactoryForLogLevel(logLevel: MinimumLogLevel, serilogLogger: serilogLogger); + _logger = loggerFactory.CreateLogger(); + } + } + /// /// Sets Static Web Apps EasyAuth as the authentication scheme for the engine. ///