diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs index da702dcf32a..06d8f0cdb0a 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; @@ -79,6 +80,10 @@ public class FunctionInvokingChatClient : DelegatingChatClient /// This component does not own the instance and should not dispose it. private readonly ActivitySource? _activitySource; + /// The to use for metrics. + /// This component does not own the instance and should not dispose it. + private readonly Meter? _meter; + /// /// Initializes a new instance of the class. /// @@ -90,6 +95,7 @@ public FunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? logge { _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; _activitySource = innerClient.GetService(); + _meter = innerClient.GetService(); FunctionInvocationServices = functionInvocationServices; } @@ -101,7 +107,8 @@ public FunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? logge invokeAgentActivity => invokeAgentActivity is not null ? invokeAgentActivity.GetCustomProperty(OpenTelemetryChatClient.SensitiveDataEnabledCustomKey) as string is OpenTelemetryChatClient.SensitiveDataEnabledTrueValue - : InnerClient.GetService()?.EnableSensitiveData is true); + : InnerClient.GetService()?.EnableSensitiveData is true, + _meter is not null ? OtelMetricHelpers.CreateGenAIExecuteToolDurationHistogram(_meter) : null); /// /// Gets or sets the for the current function invocation. diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs index 326503bc04d..ba06c48c2c7 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs @@ -22,7 +22,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating chat client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . +/// This class provides an implementation of the GenAI Semantic Conventions v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// public sealed partial class OpenTelemetryChatClient : DelegatingChatClient @@ -129,6 +129,7 @@ protected override void Dispose(bool disposing) /// public override object? GetService(Type serviceType, object? serviceKey = null) => serviceType == typeof(ActivitySource) ? _activitySource : + serviceType == typeof(Meter) ? _meter : base.GetService(serviceType, serviceKey); /// @@ -353,6 +354,24 @@ public override async IAsyncEnumerable GetStreamingResponseA _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.TopP, top_p); } + if (options.Reasoning?.Effort is ReasoningEffort reasoningEffort) + { + string? reasoningLevel = reasoningEffort switch + { + ReasoningEffort.None => "none", + ReasoningEffort.Low => "low", + ReasoningEffort.Medium => "medium", + ReasoningEffort.High => "high", + ReasoningEffort.ExtraHigh => "extrahigh", + _ => null, + }; + + if (reasoningLevel is not null) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.ReasoningLevel, reasoningLevel); + } + } + if (options.ResponseFormat is not null) { switch (options.ResponseFormat) diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs index a20b512c7b0..b431718fce0 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs @@ -20,7 +20,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating image generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . +/// This class provides an implementation of the GenAI Semantic Conventions v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// [Experimental(DiagnosticIds.Experiments.AIImageGeneration, UrlFormat = DiagnosticIds.UrlFormat)] diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs index c4a1d2448e4..9fbee90032b 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Common/FunctionInvocationProcessor.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.Metrics; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -22,6 +23,7 @@ internal sealed class FunctionInvocationProcessor { private readonly ILogger _logger; private readonly ActivitySource? _activitySource; + private readonly Histogram? _executeToolDurationHistogram; private readonly Func> _invokeFunction; private readonly Func _isSensitiveDataEnabled; @@ -36,16 +38,19 @@ internal sealed class FunctionInvocationProcessor /// Receives the invoke agent activity (or null if not in agent context). /// Returns true if sensitive data should be logged/tagged, false otherwise. /// + /// An optional histogram for recording gen_ai.execute_tool.duration metric values. public FunctionInvocationProcessor( ILogger logger, ActivitySource? activitySource, Func> invokeFunction, - Func? isSensitiveDataEnabled = null) + Func? isSensitiveDataEnabled = null, + Histogram? executeToolDurationHistogram = null) { _logger = logger; _activitySource = activitySource; _invokeFunction = invokeFunction; _isSensitiveDataEnabled = isSensitiveDataEnabled ?? (_ => false); + _executeToolDurationHistogram = executeToolDurationHistogram; } /// @@ -199,15 +204,17 @@ private async Task ProcessSingleFunctionCallAsync( } object? result = null; + string? errorType = null; try { result = await _invokeFunction(context, cancellationToken).ConfigureAwait(false); } catch (Exception e) { + errorType = e.GetType().FullName; if (activity is not null) { - _ = activity.SetTag(OpenTelemetryConsts.Error.Type, e.GetType().FullName) + _ = activity.SetTag(OpenTelemetryConsts.Error.Type, errorType) .SetStatus(ActivityStatusCode.Error, e.Message); } @@ -224,6 +231,19 @@ private async Task ProcessSingleFunctionCallAsync( } finally { + if (_executeToolDurationHistogram?.Enabled is true) + { + TagList tags = default; + tags.Add(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name); + tags.Add(OpenTelemetryConsts.GenAI.Tool.Type, OpenTelemetryConsts.ToolTypeFunction); + if (errorType is not null) + { + tags.Add(OpenTelemetryConsts.Error.Type, errorType); + } + + _executeToolDurationHistogram.Record(FunctionInvocationHelpers.GetElapsedTime(startingTimestamp).TotalSeconds, tags); + } + bool loggedResult = false; if (enableSensitiveData || traceLoggingEnabled) { diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageSerializer.cs b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageSerializer.cs index c5916379508..b1f2a6d1e0c 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageSerializer.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMessageSerializer.cs @@ -268,6 +268,7 @@ internal static string SerializeChatMessages( topLevel.Equals("image", StringComparison.OrdinalIgnoreCase) ? "image" : topLevel.Equals("audio", StringComparison.OrdinalIgnoreCase) ? "audio" : topLevel.Equals("video", StringComparison.OrdinalIgnoreCase) ? "video" : + topLevel.Equals("application", StringComparison.OrdinalIgnoreCase) ? "document" : null; } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Common/OtelMetricHelpers.cs b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMetricHelpers.cs index ca572ca515a..61ba6efa32f 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Common/OtelMetricHelpers.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Common/OtelMetricHelpers.cs @@ -23,4 +23,12 @@ public static Histogram CreateGenAIOperationDurationHistogram(Meter mete OpenTelemetryConsts.SecondsUnit, OpenTelemetryConsts.GenAI.Client.OperationDuration.Description, advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.Client.OperationDuration.ExplicitBucketBoundaries }); + + /// Creates the standard gen_ai.execute_tool.duration histogram on . + public static Histogram CreateGenAIExecuteToolDurationHistogram(Meter meter) => + meter.CreateHistogram( + OpenTelemetryConsts.GenAI.ExecuteTool.Duration.Name, + OpenTelemetryConsts.SecondsUnit, + OpenTelemetryConsts.GenAI.ExecuteTool.Duration.Description, + advice: new() { HistogramBucketBoundaries = OpenTelemetryConsts.GenAI.ExecuteTool.Duration.ExplicitBucketBoundaries }); } diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs index 090332b255f..575d7766396 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs @@ -18,7 +18,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating embedding generator that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . +/// This class provides an implementation of the GenAI Semantic Conventions v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// /// The type of input used to produce embeddings. diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index 302cfd89314..bc0e1043109 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -87,6 +87,16 @@ public static class TimePerOutputChunk } } + public static class ExecuteTool + { + public static class Duration + { + public const string Description = "Measures the duration of a single tool execution"; + public const string Name = "gen_ai.execute_tool.duration"; + public static readonly double[] ExplicitBucketBoundaries = Client.OperationDuration.ExplicitBucketBoundaries; + } + } + public static class Conversation { public const string Id = "gen_ai.conversation.id"; @@ -128,6 +138,7 @@ public static class Request public const string Model = "gen_ai.request.model"; public const string MaxTokens = "gen_ai.request.max_tokens"; public const string PresencePenalty = "gen_ai.request.presence_penalty"; + public const string ReasoningLevel = "gen_ai.request.reasoning.level"; public const string Seed = "gen_ai.request.seed"; public const string StopSequences = "gen_ai.request.stop_sequences"; public const string Stream = "gen_ai.request.stream"; diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs index 7c79bc449fb..ccce85378cf 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/FunctionInvokingRealtimeClientSession.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.Metrics; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -67,6 +68,10 @@ internal sealed class FunctionInvokingRealtimeClientSession : IRealtimeClientSes /// This component does not own the instance and should not dispose it. private readonly ActivitySource? _activitySource; + /// The to use for metrics. + /// This component does not own the instance and should not dispose it. + private readonly Meter? _meter; + /// The inner session to delegate to. private readonly IRealtimeClientSession _innerSession; @@ -86,6 +91,7 @@ public FunctionInvokingRealtimeClientSession(IRealtimeClientSession innerSession _client = Throw.IfNull(client); _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance; _activitySource = innerSession.GetService(); + _meter = innerSession.GetService(); FunctionInvocationServices = functionInvocationServices; } @@ -93,7 +99,8 @@ public FunctionInvokingRealtimeClientSession(IRealtimeClientSession innerSession private FunctionInvocationProcessor Processor => field ??= new FunctionInvocationProcessor( _logger, _activitySource, - InvokeFunctionAsync); + InvokeFunctionAsync, + executeToolDurationHistogram: _meter is not null ? OtelMetricHelpers.CreateGenAIExecuteToolDurationHistogram(_meter) : null); /// /// Gets or sets the for the current function invocation. diff --git a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs index a7f78f9e9a6..a3403e13b59 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Realtime/OpenTelemetryRealtimeClientSession.cs @@ -24,7 +24,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating realtime session that follows the OpenTelemetry Semantic Conventions for Generative AI systems where applicable. /// /// -/// This class follows the patterns of the Semantic Conventions for Generative AI systems v1.41 where applicable, as defined at +/// This class follows the patterns of the GenAI Semantic Conventions v1.41 where applicable, as defined at /// , with custom extensions for realtime-specific behavior. /// The specification does not currently define a realtime operation; a custom operation name is used. /// @@ -168,6 +168,7 @@ public async ValueTask DisposeAsync() return serviceType == typeof(ActivitySource) ? _activitySource : + serviceType == typeof(Meter) ? _meter : serviceKey is null && serviceType.IsInstanceOfType(this) ? this : _innerSession.GetService(serviceType, serviceKey); } diff --git a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs index 82ece57f673..e5c79923fa5 100644 --- a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/OpenTelemetrySpeechToTextClient.cs @@ -22,7 +22,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating speech-to-text client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . +/// This class provides an implementation of the GenAI Semantic Conventions v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// [Experimental(DiagnosticIds.Experiments.AISpeechToText, UrlFormat = DiagnosticIds.UrlFormat)] diff --git a/src/Libraries/Microsoft.Extensions.AI/TextToSpeech/OpenTelemetryTextToSpeechClient.cs b/src/Libraries/Microsoft.Extensions.AI/TextToSpeech/OpenTelemetryTextToSpeechClient.cs index 3cf4eed611d..9d57d81012b 100644 --- a/src/Libraries/Microsoft.Extensions.AI/TextToSpeech/OpenTelemetryTextToSpeechClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/TextToSpeech/OpenTelemetryTextToSpeechClient.cs @@ -21,7 +21,7 @@ namespace Microsoft.Extensions.AI; /// Represents a delegating text-to-speech client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. /// -/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . +/// This class provides an implementation of the GenAI Semantic Conventions v1.41, defined at . /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. /// [Experimental(DiagnosticIds.Experiments.AITextToSpeech, UrlFormat = DiagnosticIds.UrlFormat)] diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs index 4a88883cdbf..1f04abfa796 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs @@ -3639,5 +3639,50 @@ public async Task ServerHandledFunctionCalls_NoMatchingFRC_StillInvoked(bool str Assert.Contains(response.Messages, m => m.Contents.Any(c => c is FunctionResultContent frc2 && frc2.CallId == "callId1")); } -} + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExecuteToolDuration_MetricRecorded(bool streaming) + { + string sourceName = Guid.NewGuid().ToString(); + + ChatOptions options = new() + { + Tools = [AIFunctionFactory.Create(() => "Result 1", "Func1")] + }; + + List plan = + [ + new ChatMessage(ChatRole.User, "hello"), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("callId1", "Func1")]), + new ChatMessage(ChatRole.Tool, [new FunctionResultContent("callId1", result: "Result 1")]), + new ChatMessage(ChatRole.Assistant, "world"), + ]; + + using var metricCollector = new Microsoft.Extensions.Diagnostics.Metrics.Testing.MetricCollector(null, sourceName, "gen_ai.execute_tool.duration"); + + Func configurePipeline = b => + { + b.UseFunctionInvocation(); + b.UseOpenTelemetry(sourceName: sourceName); + return b; + }; + + if (streaming) + { + await InvokeAndAssertStreamingAsync(options, plan, configurePipeline: configurePipeline); + } + else + { + await InvokeAndAssertAsync(options, plan, configurePipeline: configurePipeline); + } + + var measurements = metricCollector.GetMeasurementSnapshot(); + var measurement = Assert.Single(measurements); + Assert.True(measurement.Value >= 0); + Assert.True(measurement.ContainsTags( + new KeyValuePair("gen_ai.tool.name", "Func1"), + new KeyValuePair("gen_ai.tool.type", "function"))); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs index 6ef724f7c66..7e8d0cae5bd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/OpenTelemetryChatClientTests.cs @@ -131,6 +131,7 @@ async static IAsyncEnumerable CallbackAsync( Temperature = 6.0f, Seed = 42, StopSequences = ["hello", "world"], + Reasoning = new ReasoningOptions { Effort = ReasoningEffort.High }, AdditionalProperties = new() { ["service_tier"] = "value1", @@ -183,6 +184,7 @@ async static IAsyncEnumerable CallbackAsync( Assert.Equal(enableSensitiveData ? "value1" : null, activity.GetTagItem("service_tier")); Assert.Equal(enableSensitiveData ? "value2" : null, activity.GetTagItem("SomethingElse")); Assert.Equal(42L, activity.GetTagItem("gen_ai.request.seed")); + Assert.Equal("high", activity.GetTagItem("gen_ai.request.reasoning.level")); Assert.Equal("id123", activity.GetTagItem("gen_ai.response.id")); Assert.Equal("""["stop"]""", activity.GetTagItem("gen_ai.response.finish_reasons")); @@ -445,6 +447,7 @@ async static IAsyncEnumerable CallbackAsync( new TextReasoningContent("User reasoning"), new DataContent(Convert.FromBase64String("ZGF0YSBjb250ZW50"), "audio/mp3"), new UriContent(new Uri("https://example.com/video.mp4"), "video/mp4"), + new DataContent(Convert.FromBase64String("cGRmY29udGVudA=="), "application/pdf"), new HostedFileContent("file-xyz789"), ]), new(ChatRole.Assistant, [new FunctionCallContent("call-456", "SearchFiles")]), @@ -492,6 +495,12 @@ async static IAsyncEnumerable CallbackAsync( "mime_type": "video/mp4", "modality": "video" }, + { + "type": "blob", + "content": "cGRmY29udGVudA==", + "mime_type": "application/pdf", + "modality": "document" + }, { "type": "file", "file_id": "file-xyz789"