Cherry-Pick File Sink & Bug Fixes for 1.6 Release

schemas/dab.draft.schema.json

@@ -483,6 +483,50 @@
                 "required": [ "auth" ]
               }
             },
+            "file": {
+              "type": "object",
+              "additionalProperties": false,
+              "properties": {
+                "enabled": {
+                  "type": "boolean",
+                  "description": "Enable/disable file sink telemetry logging.",
+                  "default": false
+                },
+                "path": {
+                  "type": "string",
+                  "description": "File path for telemetry logs.",
+                  "default": "/logs/dab-log.txt"
+                },
+                "rolling-interval": {
+                  "type": "string",
+                  "description": "Rolling interval for log files.",
+                  "default": "Day",
+                  "enum": [ "Minute", "Hour", "Day", "Month", "Year", "Infinite" ]
+                },
+                "retained-file-count-limit": {
+                  "type": "integer",
+                  "description": "Maximum number of retained log files.",
+                  "default": 1,
+                  "minimum": 1
+                },
+                "file-size-limit-bytes": {
+                  "type": "integer",
+                  "description": "Maximum file size in bytes before rolling.",
+                  "default": 1048576,
+                  "minimum": 1
+                }
+              },
+              "if": {
+                "properties": {
+                  "enabled": {
+                    "const": true
+                  }
+                }
+              },
+              "then": {
+                "required": [ "path" ]
+              }
+            },
             "log-level": {
               "type": "object",
               "description": "Global configuration of log level, defines logging severity levels for specific classes, when 'null' it will set logging level based on 'host: mode' property",

src/Cli.Tests/ConfigureOptionsTests.cs

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using Serilog;
+
 namespace Cli.Tests
 {
     /// <summary>
@@ -188,6 +190,49 @@ public void TestAddAzureLogAnalyticsOptions()
             Assert.AreEqual("dce-endpoint-test", config.Runtime.Telemetry.AzureLogAnalytics.Auth.DceEndpoint);
         }
 
+        /// <summary>
+        /// Tests that running the "configure --file" commands on a config without file sink properties results
+        /// in a valid config being generated.
+        /// </summary>
+        [TestMethod]
+        public void TestAddFileSinkOptions()
+        {
+            // Arrange
+            string fileSinkPath = "/custom/log/path.txt";
+            RollingInterval fileSinkRollingInterval = RollingInterval.Hour;
+            int fileSinkRetainedFileCountLimit = 5;
+            int fileSinkFileSizeLimitBytes = 2097152;
+
+            _fileSystem!.AddFile(TEST_RUNTIME_CONFIG_FILE, new MockFileData(INITIAL_CONFIG));
+
+            Assert.IsTrue(_fileSystem!.File.Exists(TEST_RUNTIME_CONFIG_FILE));
+
+            // Act: Attempts to add file options
+            ConfigureOptions options = new(
+                fileSinkEnabled: CliBool.True,
+                fileSinkPath: fileSinkPath,
+                fileSinkRollingInterval: fileSinkRollingInterval,
+                fileSinkRetainedFileCountLimit: fileSinkRetainedFileCountLimit,
+                fileSinkFileSizeLimitBytes: fileSinkFileSizeLimitBytes,
+                config: TEST_RUNTIME_CONFIG_FILE
+            );
+
+            bool isSuccess = TryConfigureSettings(options, _runtimeConfigLoader!, _fileSystem!);
+
+            // Assert: Validate the file options are added.
+            Assert.IsTrue(isSuccess);
+            string updatedConfig = _fileSystem!.File.ReadAllText(TEST_RUNTIME_CONFIG_FILE);
+            Assert.IsTrue(RuntimeConfigLoader.TryParseConfig(updatedConfig, out RuntimeConfig? config));
+            Assert.IsNotNull(config.Runtime);
+            Assert.IsNotNull(config.Runtime.Telemetry);
+            Assert.IsNotNull(config.Runtime.Telemetry.File);
+            Assert.AreEqual(true, config.Runtime.Telemetry.File.Enabled);
+            Assert.AreEqual(fileSinkPath, config.Runtime.Telemetry.File.Path);
+            Assert.AreEqual(fileSinkRollingInterval.ToString(), config.Runtime.Telemetry.File.RollingInterval);
+            Assert.AreEqual(fileSinkRetainedFileCountLimit, config.Runtime.Telemetry.File.RetainedFileCountLimit);
+            Assert.AreEqual(fileSinkFileSizeLimitBytes, config.Runtime.Telemetry.File.FileSizeLimitBytes);
+        }
+
         /// <summary>
         /// Tests that running "dab configure --runtime.graphql.enabled" on a config with various values results
         /// in runtime. Takes in updated value for graphql.enabled and 

src/Cli.Tests/ValidateConfigTests.cs

@@ -3,6 +3,7 @@
 
 using Azure.DataApiBuilder.Core.Configurations;
 using Azure.DataApiBuilder.Core.Models;
+using Serilog;
 
 namespace Cli.Tests;
 /// <summary>
@@ -282,17 +283,6 @@ public void ValidateConfigSchemaWhereConfigReferencesEnvironmentVariables()
     public async Task TestValidateAKVOptionsWithoutEndpointFails()
     {
         // Arrange
-        _fileSystem!.AddFile(TEST_RUNTIME_CONFIG_FILE, new MockFileData(INITIAL_CONFIG));
-        Assert.IsTrue(_fileSystem!.File.Exists(TEST_RUNTIME_CONFIG_FILE));
-        Mock<RuntimeConfigProvider> mockRuntimeConfigProvider = new(_runtimeConfigLoader);
-        RuntimeConfigValidator validator = new(mockRuntimeConfigProvider.Object, _fileSystem, new Mock<ILogger<RuntimeConfigValidator>>().Object);
-        Mock<ILoggerFactory> mockLoggerFactory = new();
-        Mock<ILogger<JsonConfigSchemaValidator>> mockLogger = new();
-        mockLoggerFactory
-            .Setup(factory => factory.CreateLogger(typeof(JsonConfigSchemaValidator).FullName!))
-            .Returns(mockLogger.Object);
-
-        // Act: Attempts to add AKV options
         ConfigureOptions options = new(
             azureKeyVaultRetryPolicyMaxCount: 1,
             azureKeyVaultRetryPolicyDelaySeconds: 1,
@@ -302,14 +292,8 @@ public async Task TestValidateAKVOptionsWithoutEndpointFails()
             config: TEST_RUNTIME_CONFIG_FILE
         );
 
-        bool isSuccess = TryConfigureSettings(options, _runtimeConfigLoader!, _fileSystem!);
-
-        // Assert: Settings are configured, config parses, validation fails.
-        Assert.IsTrue(isSuccess);
-        string updatedConfig = _fileSystem!.File.ReadAllText(TEST_RUNTIME_CONFIG_FILE);
-        Assert.IsTrue(RuntimeConfigLoader.TryParseConfig(updatedConfig, out RuntimeConfig? config));
-        JsonSchemaValidationResult result = await validator.ValidateConfigSchema(config, TEST_RUNTIME_CONFIG_FILE, mockLoggerFactory.Object);
-        Assert.IsFalse(result.IsValid);
+        // Act
+        await ValidatePropertyOptionsFails(options);
     }
 
     /// <summary>
@@ -319,24 +303,53 @@ public async Task TestValidateAKVOptionsWithoutEndpointFails()
     public async Task TestValidateAzureLogAnalyticsOptionsWithoutAuthFails()
     {
         // Arrange
+        ConfigureOptions options = new(
+            azureLogAnalyticsEnabled: CliBool.True,
+            azureLogAnalyticsDabIdentifier: "dab-identifier-test",
+            azureLogAnalyticsFlushIntervalSeconds: 1,
+            config: TEST_RUNTIME_CONFIG_FILE
+        );
+
+        // Act
+        await ValidatePropertyOptionsFails(options);
+    }
+
+    /// <summary>
+    /// Tests that validation fails when File Sink options are configured without the 'path' property.
+    /// </summary>
+    [TestMethod]
+    public async Task TestValidateFileSinkOptionsWithoutPathFails()
+    {
+        // Arrange
+        ConfigureOptions options = new(
+            fileSinkEnabled: CliBool.True,
+            fileSinkRollingInterval: RollingInterval.Day,
+            fileSinkRetainedFileCountLimit: 1,
+            fileSinkFileSizeLimitBytes: 1024,
+            config: TEST_RUNTIME_CONFIG_FILE
+        );
+
+        // Act
+        await ValidatePropertyOptionsFails(options);
+    }
+
+    /// <summary>
+    /// Helper function that ensures properties with missing options fail validation.
+    /// </summary>
+    private async Task ValidatePropertyOptionsFails(ConfigureOptions options)
+    {
         _fileSystem!.AddFile(TEST_RUNTIME_CONFIG_FILE, new MockFileData(INITIAL_CONFIG));
         Assert.IsTrue(_fileSystem!.File.Exists(TEST_RUNTIME_CONFIG_FILE));
         Mock<RuntimeConfigProvider> mockRuntimeConfigProvider = new(_runtimeConfigLoader);
         RuntimeConfigValidator validator = new(mockRuntimeConfigProvider.Object, _fileSystem, new Mock<ILogger<RuntimeConfigValidator>>().Object);
+
         Mock<ILoggerFactory> mockLoggerFactory = new();
         Mock<ILogger<JsonConfigSchemaValidator>> mockLogger = new();
         mockLoggerFactory
             .Setup(factory => factory.CreateLogger(typeof(JsonConfigSchemaValidator).FullName!))
             .Returns(mockLogger.Object);
 
-        // Act: Attempts to add Azure Log Analytics options without Auth options
-        ConfigureOptions options = new(
-            azureLogAnalyticsEnabled: CliBool.True,
-            azureLogAnalyticsDabIdentifier: "dab-identifier-test",
-            azureLogAnalyticsFlushIntervalSeconds: 1,
-            config: TEST_RUNTIME_CONFIG_FILE
-        );
-
+        // Act: Attempts to add File Sink options without empty path
         bool isSuccess = TryConfigureSettings(options, _runtimeConfigLoader!, _fileSystem!);
 
         // Assert: Settings are configured, config parses, validation fails.

src/Cli/Commands/ConfigureOptions.cs

@@ -8,7 +8,9 @@
 using Cli.Constants;
 using CommandLine;
 using Microsoft.Extensions.Logging;
+using Serilog;
 using static Cli.Utils;
+using ILogger = Microsoft.Extensions.Logging.ILogger;
 
 namespace Cli.Commands
 {
@@ -54,6 +56,11 @@ public ConfigureOptions(
             string? azureLogAnalyticsCustomTableName = null,
             string? azureLogAnalyticsDcrImmutableId = null,
             string? azureLogAnalyticsDceEndpoint = null,
+            CliBool? fileSinkEnabled = null,
+            string? fileSinkPath = null,
+            RollingInterval? fileSinkRollingInterval = null,
+            int? fileSinkRetainedFileCountLimit = null,
+            long? fileSinkFileSizeLimitBytes = null,
             string? config = null)
             : base(config)
         {
@@ -98,6 +105,12 @@ public ConfigureOptions(
             AzureLogAnalyticsCustomTableName = azureLogAnalyticsCustomTableName;
             AzureLogAnalyticsDcrImmutableId = azureLogAnalyticsDcrImmutableId;
             AzureLogAnalyticsDceEndpoint = azureLogAnalyticsDceEndpoint;
+            // File
+            FileSinkEnabled = fileSinkEnabled;
+            FileSinkPath = fileSinkPath;
+            FileSinkRollingInterval = fileSinkRollingInterval;
+            FileSinkRetainedFileCountLimit = fileSinkRetainedFileCountLimit;
+            FileSinkFileSizeLimitBytes = fileSinkFileSizeLimitBytes;
         }
 
         [Option("data-source.database-type", Required = false, HelpText = "Database type. Allowed values: MSSQL, PostgreSQL, CosmosDB_NoSQL, MySQL.")]
@@ -202,6 +215,21 @@ public ConfigureOptions(
         [Option("runtime.telemetry.azure-log-analytics.auth.dce-endpoint", Required = false, HelpText = "Configure DCE Endpoint for Azure Log Analytics to find table to send telemetry data")]
         public string? AzureLogAnalyticsDceEndpoint { get; }
 
+        [Option("runtime.telemetry.file.enabled", Required = false, HelpText = "Enable/Disable File Sink logging. Default: False (boolean)")]
+        public CliBool? FileSinkEnabled { get; }
+
+        [Option("runtime.telemetry.file.path", Required = false, HelpText = "Configure path for File Sink logging. Default: /logs/dab-log.txt")]
+        public string? FileSinkPath { get; }
+
+        [Option("runtime.telemetry.file.rolling-interval", Required = false, HelpText = "Configure rolling interval for File Sink logging. Default: Day")]
+        public RollingInterval? FileSinkRollingInterval { get; }
+
+        [Option("runtime.telemetry.file.retained-file-count-limit", Required = false, HelpText = "Configure maximum number of retained files. Default: 1")]
+        public int? FileSinkRetainedFileCountLimit { get; }
+
+        [Option("runtime.telemetry.file.file-size-limit-bytes", Required = false, HelpText = "Configure maximum file size limit in bytes. Default: 1048576")]
+        public long? FileSinkFileSizeLimitBytes { get; }
+
         public int Handler(ILogger logger, FileSystemRuntimeConfigLoader loader, IFileSystem fileSystem)
         {
             logger.LogInformation("{productName} {version}", PRODUCT_NAME, ProductInfo.GetProductVersion());

src/Cli/ConfigGenerator.cs

@@ -13,6 +13,7 @@
 using Azure.DataApiBuilder.Service;
 using Cli.Commands;
 using Microsoft.Extensions.Logging;
+using Serilog;
 using static Cli.Utils;
 
 namespace Cli
@@ -798,6 +799,25 @@ options.AzureLogAnalyticsDcrImmutableId is not null ||
                 }
             }
 
+            // Telemetry: File Sink
+            if (options.FileSinkEnabled is not null ||
+                options.FileSinkPath is not null ||
+                options.FileSinkRollingInterval is not null ||
+                options.FileSinkRetainedFileCountLimit is not null ||
+                options.FileSinkFileSizeLimitBytes is not null)
+            {
+                FileSinkOptions updatedFileSinkOptions = runtimeConfig?.Runtime?.Telemetry?.File ?? new();
+                bool status = TryUpdateConfiguredFileOptions(options, ref updatedFileSinkOptions);
+                if (status)
+                {
+                    runtimeConfig = runtimeConfig! with { Runtime = runtimeConfig.Runtime! with { Telemetry = runtimeConfig.Runtime!.Telemetry is not null ? runtimeConfig.Runtime!.Telemetry with { File = updatedFileSinkOptions } : new TelemetryOptions(File: updatedFileSinkOptions) } };
+                }
+                else
+                {
+                    return false;
+                }
+            }
+
             return runtimeConfig != null;
         }
 
@@ -1199,6 +1219,74 @@ private static bool TryUpdateConfiguredAzureLogAnalyticsOptions(
             }
         }
 
+        /// <summary>
+        /// Updates the file sink options in the configuration.
+        /// </summary>
+        /// <param name="options">The configuration options provided by the user.</param>
+        /// <param name="fileOptions">The file sink options to be updated.</param>
+        /// <returns>True if the options were successfully updated; otherwise, false.</returns>
+        private static bool TryUpdateConfiguredFileOptions(
+            ConfigureOptions options,
+            ref FileSinkOptions fileOptions)
+        {
+            try
+            {
+                // Runtime.Telemetry.File.Enabled
+                if (options.FileSinkEnabled is not null)
+                {
+                    fileOptions = fileOptions with { Enabled = options.FileSinkEnabled is CliBool.True, UserProvidedEnabled = true };
+                    _logger.LogInformation($"Updated configuration with runtime.telemetry.file.enabled as '{options.FileSinkEnabled}'");
+                }
+
+                // Runtime.Telemetry.File.Path
+                if (options.FileSinkPath is not null)
+                {
+                    fileOptions = fileOptions with { Path = options.FileSinkPath, UserProvidedPath = true };
+                    _logger.LogInformation($"Updated configuration with runtime.telemetry.file.path as '{options.FileSinkPath}'");
+                }
+
+                // Runtime.Telemetry.File.RollingInterval
+                if (options.FileSinkRollingInterval is not null)
+                {
+                    fileOptions = fileOptions with { RollingInterval = ((RollingInterval)options.FileSinkRollingInterval).ToString(), UserProvidedRollingInterval = true };
+                    _logger.LogInformation($"Updated configuration with runtime.telemetry.file.rolling-interval as '{options.FileSinkRollingInterval}'");
+                }
+
+                // Runtime.Telemetry.File.RetainedFileCountLimit
+                if (options.FileSinkRetainedFileCountLimit is not null)
+                {
+                    if (options.FileSinkRetainedFileCountLimit <= 0)
+                    {
+                        _logger.LogError("Failed to update configuration with runtime.telemetry.file.retained-file-count-limit. Value must be a positive integer greater than 0.");
+                        return false;
+                    }
+
+                    fileOptions = fileOptions with { RetainedFileCountLimit = (int)options.FileSinkRetainedFileCountLimit, UserProvidedRetainedFileCountLimit = true };
+                    _logger.LogInformation($"Updated configuration with runtime.telemetry.file.retained-file-count-limit as '{options.FileSinkRetainedFileCountLimit}'");
+                }
+
+                // Runtime.Telemetry.File.FileSizeLimitBytes
+                if (options.FileSinkFileSizeLimitBytes is not null)
+                {
+                    if (options.FileSinkFileSizeLimitBytes <= 0)
+                    {
+                        _logger.LogError("Failed to update configuration with runtime.telemetry.file.file-size-limit-bytes. Value must be a positive integer greater than 0.");
+                        return false;
+                    }
+
+                    fileOptions = fileOptions with { FileSizeLimitBytes = (long)options.FileSinkFileSizeLimitBytes, UserProvidedFileSizeLimitBytes = true };
+                    _logger.LogInformation($"Updated configuration with runtime.telemetry.file.file-size-limit-bytes as '{options.FileSinkFileSizeLimitBytes}'");
+                }
+
+                return true;
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError($"Failed to update configuration with runtime.telemetry.file. Exception message: {ex.Message}.");
+                return false;
+            }
+        }
+
         /// <summary>
         /// Parse permission string to create PermissionSetting array.
         /// </summary>

src/Config/Azure.DataApiBuilder.Config.csproj

@@ -18,14 +18,15 @@
       <PackageReference Include="Microsoft.AspNetCore.Authorization" />
       <PackageReference Include="Microsoft.IdentityModel.Protocols" />
       <PackageReference Include="Microsoft.IdentityModel.Protocols.OpenIdConnect" />
+      <PackageReference Include="Serilog.Sinks.File" />
       <PackageReference Include="System.IO.Abstractions" />
       <PackageReference Include="System.Drawing.Common" />
       <PackageReference Include="Microsoft.Data.SqlClient" />
       <PackageReference Include="Microsoft.IdentityModel.JsonWebTokens" />
       <PackageReference Include="System.IdentityModel.Tokens.Jwt" />
       <PackageReference Include="Humanizer" />
       <PackageReference Include="Npgsql" />
-      <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol"  />
+      <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
   </ItemGroup>
 
     <ItemGroup>