diff --git a/Appraisals b/Appraisals index f953e00be..de459c5b2 100644 --- a/Appraisals +++ b/Appraisals @@ -1,25 +1,25 @@ # frozen_string_literal: true appraise 'rails-7.1' do - group :development do + group :development, :test do gem 'rails', '~> 7.1.0' end end appraise 'rails-7.2' do - group :development do + group :development, :test do gem 'rails', '~> 7.2.0' end end appraise 'rails-8.0' do - group :development do + group :development, :test do gem 'rails', '~> 8.0.0' end end appraise 'rails-8.1' do - group :development do + group :development, :test do gem 'rails', '~> 8.1.0' end end diff --git a/Gemfile b/Gemfile index b9a7dcc19..c95952ce5 100644 --- a/Gemfile +++ b/Gemfile @@ -4,14 +4,14 @@ source 'https://rubygems.org' gemspec -group :development do # rubocop:disable Metrics/BlockLength +group :development, :test do # rubocop:disable Metrics/BlockLength gem 'appraisal' gem 'async', platform: :mri gem 'bundler', '>= 2.0' gem 'codecov' gem 'dotenv' gem 'ferrum' - gem 'flay' + gem 'flay', '< 2.14' # 2.14 switched from ruby_parser to prism, causing CI issues gem 'image_processing', '~> 1.2' gem 'irb' gem 'json-schema' @@ -39,4 +39,7 @@ group :development do # rubocop:disable Metrics/BlockLength # Optional dependency for Vertex AI gem 'googleauth' + + # OpenTelemetry for observability testing + gem 'opentelemetry-sdk' end diff --git a/docs/_advanced/observability.md b/docs/_advanced/observability.md new file mode 100644 index 000000000..42119284b --- /dev/null +++ b/docs/_advanced/observability.md @@ -0,0 +1,330 @@ +--- +layout: default +title: Observability +nav_order: 7 +description: Send traces to LangSmith, DataDog, or any OpenTelemetry backend. Monitor your LLM usage in production. +redirect_from: + - /guides/observability +--- + +# {{ page.title }} +{: .no_toc } + +{{ page.description }} +{: .fs-6 .fw-300 } + +## Table of contents +{: .no_toc .text-delta } + +1. TOC +{:toc} + +--- + +After reading this guide, you will know: + +* How to enable OpenTelemetry tracing in RubyLLM +* How to configure backends like LangSmith, DataDog, and Jaeger +* How streaming and non-streaming requests are traced +* How session tracking groups multi-turn conversations +* How to add custom metadata to traces +* What attributes are captured in spans + +## What's Supported + +| Feature | Status | +|---------|--------| +| Chat completions | ✅ Supported | +| Streaming | ✅ Supported | +| Tool calls | ✅ Supported | +| Session tracking | ✅ Supported | +| Content logging (opt-in) | ✅ Supported | +| Embeddings | ❌ Not yet supported | +| Image generation | ❌ Not yet supported | +| Transcription | ❌ Not yet supported | +| Moderation | ❌ Not yet supported | + +--- + +## Quick Start + +### 1. Install OpenTelemetry gems + +```ruby +# Gemfile +gem 'opentelemetry-sdk' +gem 'opentelemetry-exporter-otlp' +``` + +### 2. Enable tracing in RubyLLM + +```ruby +RubyLLM.configure do |config| + config.tracing_enabled = true +end +``` + +### 3. Configure your exporter + +See [Backend Setup](#backend-setup) for LangSmith, DataDog, Jaeger, etc. + +--- + +## Configuration Options + +```ruby +RubyLLM.configure do |config| + # Enable tracing (default: false) + config.tracing_enabled = true + + # Log prompt/completion content (default: false) + config.tracing_log_content = true + + # Max content length before truncation (default: 10000) + config.tracing_max_content_length = 5000 + + # Enable LangSmith-specific span attributes (default: false) + config.tracing_langsmith_compat = true +end +``` + +> **Privacy note:** `tracing_log_content` sends your prompts and completions to your tracing backend. Only enable this if you're comfortable with your backend seeing this data. +{: .warning } + +### Service Name + +Your service name identifies your application in the tracing backend. Set it via environment variable: + +```bash +export OTEL_SERVICE_NAME="my_app" +``` + +### Custom Metadata + +You can attach custom metadata to traces for filtering and debugging: + +```ruby +chat = RubyLLM.chat + .with_metadata(user_id: current_user.id, request_id: request.uuid) +chat.ask("Hello!") +``` + +Metadata appears as `metadata.*` attributes by default. When `tracing_langsmith_compat` is enabled, metadata uses the `langsmith.metadata.*` prefix for proper LangSmith panel integration. + +You can also set a custom prefix: + +```ruby +RubyLLM.configure do |config| + config.tracing_metadata_prefix = 'app.metadata' +end +``` + +--- + +## Backend Setup + +### LangSmith + +LangSmith is LangChain's observability platform with specialized LLM debugging features. + +```ruby +# config/initializers/ruby_llm.rb +RubyLLM.configure do |config| + config.tracing_enabled = true + config.tracing_log_content = true + config.tracing_langsmith_compat = true # Adds LangSmith-specific span attributes +end +``` + +```ruby +# config/initializers/opentelemetry.rb +require 'opentelemetry/sdk' +require 'opentelemetry/exporter/otlp' + +OpenTelemetry::SDK.configure do |c| + c.add_span_processor( + OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: 'https://api.smith.langchain.com/otel/v1/traces', + headers: { + 'x-api-key' => 'lsv2_pt_...', + 'Langsmith-Project' => 'my-project' + } + ) + ) + ) +end +``` + +LangSmith uses the `Langsmith-Project` header (not `service_name`) to organize traces. + +When `tracing_langsmith_compat = true`, RubyLLM adds these additional attributes for LangSmith integration: +- `langsmith.span.kind` - Identifies span type (LLM, TOOL) +- `input.value` / `output.value` - Populates LangSmith's Input/Output panels +- `langsmith.metadata.*` - Custom metadata appears in LangSmith's metadata panel + +### Other Backends + +RubyLLM works with any OpenTelemetry-compatible backend. Configure the `opentelemetry-exporter-otlp` gem to send traces to your platform of choice. + +> Using DataDog, Jaeger, Honeycomb, or another platform? Consider [contributing](https://github.com/crmne/ruby_llm/blob/main/CONTRIBUTING.md) a setup example! +{: .note } + +--- + +## What Gets Traced + +RubyLLM follows the [OpenTelemetry Semantic Conventions for GenAI](https://opentelemetry.io/docs/specs/semconv/gen-ai/) ([GitHub source](https://github.com/open-telemetry/semantic-conventions/tree/main/docs/gen-ai)). + +### Chat Completions + +Each call to `chat.ask()` creates a `chat {model_id}` span (e.g., `chat gpt-4o`) with: + +| Attribute | Description | +|-----------|-------------| +| `gen_ai.provider.name` | Provider name (openai, anthropic, etc.) | +| `gen_ai.operation.name` | Set to `chat` | +| `gen_ai.request.model` | Requested model ID | +| `gen_ai.request.temperature` | Temperature setting (if specified) | +| `gen_ai.response.model` | Actual model used | +| `gen_ai.usage.input_tokens` | Input token count | +| `gen_ai.usage.output_tokens` | Output token count | +| `gen_ai.conversation.id` | Session ID for grouping conversations | + +When `tracing_langsmith_compat = true`, additional attributes are added: + +| Attribute | Description | +|-----------|-------------| +| `langsmith.span.kind` | Set to `LLM` | +| `input.value` | Last user message (for LangSmith Input panel) | +| `output.value` | Assistant response (for LangSmith Output panel) | + +### Streaming + +Streaming responses are traced identically to non-streaming responses. The span wraps the entire streaming operation: + +```ruby +chat.ask("Write a poem") do |chunk| + print chunk.content # Chunks stream in real-time +end +# Span completes here with full token counts +``` + +**How it works:** + +1. Span starts when `ask()` is called +2. Chunks stream to your block as they arrive +3. RubyLLM aggregates chunks internally +4. When streaming completes, token counts and final content are recorded on the span + +This follows the industry standard (LangSmith, Vercel AI SDK) where streaming operations get a single span representing the full request, not per-chunk spans. Tool calls during streaming create child spans just like non-streaming. + +### Tool Calls + +When tools are invoked, child `execute_tool {tool_name}` spans (e.g., `execute_tool get_weather`) are created with: + +| Attribute | Description | +|-----------|-------------| +| `gen_ai.tool.name` | Name of the tool | +| `gen_ai.tool.call.id` | Unique call identifier | +| `gen_ai.conversation.id` | Session ID for grouping | +| `gen_ai.tool.call.arguments` | Tool arguments (if content logging enabled) | +| `gen_ai.tool.call.result` | Tool result (if content logging enabled) | + +When `tracing_langsmith_compat = true`, additional attributes are added: + +| Attribute | Description | +|-----------|-------------| +| `langsmith.span.kind` | Set to `TOOL` | +| `input.value` | Tool arguments (for LangSmith Input panel) | +| `output.value` | Tool result (for LangSmith Output panel) | + +### Content Logging + +When `tracing_log_content = true`, prompts and completions are logged as JSON arrays following the OTEL GenAI spec: + +| Attribute | Description | +|-----------|-------------| +| `gen_ai.input.messages` | JSON array of input messages with role and parts | +| `gen_ai.output.messages` | JSON array of output messages with role and parts | + +Example `gen_ai.input.messages`: +```json +[ + {"role": "system", "parts": [{"type": "text", "content": "You are helpful"}]}, + {"role": "user", "parts": [{"type": "text", "content": "Hello"}]} +] +``` + +--- + +## Session Tracking + +Each `Chat` instance gets a unique `session_id`. All traces from that chat include this ID: + +```ruby +chat = RubyLLM.chat +chat.ask("Hello") # session_id: f47ac10b-58cc-4372-a567-0e02b2c3d479 +chat.ask("How are you?") # session_id: f47ac10b-58cc-4372-a567-0e02b2c3d479 (same) + +chat2 = RubyLLM.chat +chat2.ask("Hi") # session_id: 7c9e6679-7425-40de-944b-e07fc1f90ae7 (different) +``` + +### Custom Session IDs + +For applications that persist conversations, pass your own session ID to group related traces: + +```ruby +chat = RubyLLM.chat(session_id: conversation.id) +chat.ask("Hello") + +# Later, when user continues the conversation: +chat = RubyLLM.chat(session_id: conversation.id) +chat.ask("Follow up") # Same session_id, grouped together +``` + +--- + +## Troubleshooting + +### "I don't see any traces" + +1. Verify `config.tracing_enabled = true` is set +2. Check your OpenTelemetry exporter configuration +3. Ensure the `opentelemetry-sdk` gem is installed +4. Check your backend's API key and endpoint + +### "I see traces but no content" + +Enable content logging: + +```ruby +RubyLLM.configure do |config| + config.tracing_log_content = true +end +``` + +### "My tracing backend is getting too much data" + +1. Reduce `tracing_max_content_length` to truncate large messages +2. Disable content logging: `config.tracing_log_content = false` +3. Configure sampling via environment variables: + +```bash +# Sample only 10% of traces +export OTEL_TRACES_SAMPLER="traceidratio" +export OTEL_TRACES_SAMPLER_ARG="0.1" +``` + +### "Traces aren't grouped in LangSmith" + +Make sure you're reusing the same `Chat` instance for multi-turn conversations. Each `Chat.new` creates a new session. + +## Next Steps + +* [Chatting with AI Models]({% link _core_features/chat.md %}) +* [Using Tools]({% link _core_features/tools.md %}) +* [Rails Integration]({% link _advanced/rails.md %}) +* [Error Handling]({% link _advanced/error-handling.md %}) + diff --git a/gemfiles/rails_7.1.gemfile b/gemfiles/rails_7.1.gemfile index 10ea3193d..5ebf2436c 100644 --- a/gemfiles/rails_7.1.gemfile +++ b/gemfiles/rails_7.1.gemfile @@ -2,14 +2,14 @@ source "https://rubygems.org" -group :development do +group :development, :test do gem "appraisal" gem "async", platform: :mri gem "bundler", ">= 2.0" gem "codecov" gem "dotenv" gem "ferrum" - gem "flay" + gem "flay", "< 2.14" gem "image_processing", "~> 1.2" gem "irb" gem "json-schema" @@ -32,6 +32,7 @@ group :development do gem "vcr" gem "webmock", "~> 3.18" gem "googleauth" + gem "opentelemetry-sdk" end gemspec path: "../" diff --git a/gemfiles/rails_7.2.gemfile b/gemfiles/rails_7.2.gemfile index f0f87e803..c4b3915ee 100644 --- a/gemfiles/rails_7.2.gemfile +++ b/gemfiles/rails_7.2.gemfile @@ -2,14 +2,14 @@ source "https://rubygems.org" -group :development do +group :development, :test do gem "appraisal" gem "async", platform: :mri gem "bundler", ">= 2.0" gem "codecov" gem "dotenv" gem "ferrum" - gem "flay" + gem "flay", "< 2.14" gem "image_processing", "~> 1.2" gem "irb" gem "json-schema" @@ -32,6 +32,7 @@ group :development do gem "vcr" gem "webmock", "~> 3.18" gem "googleauth" + gem "opentelemetry-sdk" end gemspec path: "../" diff --git a/gemfiles/rails_8.0.gemfile b/gemfiles/rails_8.0.gemfile index 80e2c2c51..a8286fefb 100644 --- a/gemfiles/rails_8.0.gemfile +++ b/gemfiles/rails_8.0.gemfile @@ -2,14 +2,14 @@ source "https://rubygems.org" -group :development do +group :development, :test do gem "appraisal" gem "async", platform: :mri gem "bundler", ">= 2.0" gem "codecov" gem "dotenv" gem "ferrum" - gem "flay" + gem "flay", "< 2.14" gem "image_processing", "~> 1.2" gem "irb" gem "json-schema" @@ -32,6 +32,7 @@ group :development do gem "vcr" gem "webmock", "~> 3.18" gem "googleauth" + gem "opentelemetry-sdk" end gemspec path: "../" diff --git a/gemfiles/rails_8.1.gemfile b/gemfiles/rails_8.1.gemfile index b7dc8724c..257f736c9 100644 --- a/gemfiles/rails_8.1.gemfile +++ b/gemfiles/rails_8.1.gemfile @@ -2,14 +2,14 @@ source "https://rubygems.org" -group :development do +group :development, :test do gem "appraisal" gem "async", platform: :mri gem "bundler", ">= 2.0" gem "codecov" gem "dotenv" gem "ferrum" - gem "flay" + gem "flay", "< 2.14" gem "image_processing", "~> 1.2" gem "irb" gem "json-schema" @@ -32,6 +32,7 @@ group :development do gem "vcr" gem "webmock", "~> 3.18" gem "googleauth" + gem "opentelemetry-sdk" end gemspec path: "../" diff --git a/lib/ruby_llm/active_record/acts_as_legacy.rb b/lib/ruby_llm/active_record/acts_as_legacy.rb index 97679c126..f9490d64a 100644 --- a/lib/ruby_llm/active_record/acts_as_legacy.rb +++ b/lib/ruby_llm/active_record/acts_as_legacy.rb @@ -88,10 +88,12 @@ module ChatLegacyMethods def to_llm(context: nil) # model_id is a string that RubyLLM can resolve + # session_id namespaced with class name for uniqueness across models + session = "#{self.class.name}:#{id}" @chat ||= if context - context.chat(model: model_id) + context.chat(model: model_id, session_id: session) else - RubyLLM.chat(model: model_id) + RubyLLM.chat(model: model_id, session_id: session) end @chat.reset_messages! diff --git a/lib/ruby_llm/active_record/chat_methods.rb b/lib/ruby_llm/active_record/chat_methods.rb index 41930548c..fa9bea11d 100644 --- a/lib/ruby_llm/active_record/chat_methods.rb +++ b/lib/ruby_llm/active_record/chat_methods.rb @@ -79,7 +79,8 @@ def to_llm model_record = model_association @chat ||= (context || RubyLLM).chat( model: model_record.model_id, - provider: model_record.provider.to_sym + provider: model_record.provider.to_sym, + session_id: "#{self.class.name}:#{id}" # Namespaced for uniqueness across models ) @chat.reset_messages! diff --git a/lib/ruby_llm/chat.rb b/lib/ruby_llm/chat.rb index d03d872ca..8361c9e41 100644 --- a/lib/ruby_llm/chat.rb +++ b/lib/ruby_llm/chat.rb @@ -5,9 +5,13 @@ module RubyLLM class Chat include Enumerable - attr_reader :model, :messages, :tools, :params, :headers, :schema + attr_reader :model, :messages, :tools, :params, :headers, :schema, :session_id - def initialize(model: nil, provider: nil, assume_model_exists: false, context: nil) + def metadata + @metadata.dup + end + + def initialize(model: nil, provider: nil, assume_model_exists: false, context: nil, session_id: nil) if assume_model_exists && !provider raise ArgumentError, 'Provider must be specified if assume_model_exists is true' end @@ -16,6 +20,8 @@ def initialize(model: nil, provider: nil, assume_model_exists: false, context: n @config = context&.config || RubyLLM.config model_id = model || @config.default_model with_model(model_id, provider: provider, assume_exists: assume_model_exists) + @session_id = session_id || SecureRandom.uuid + @metadata = {} @temperature = nil @messages = [] @tools = {} @@ -84,6 +90,11 @@ def with_headers(**headers) self end + def with_metadata(**metadata) + @metadata = @metadata.merge(metadata) + self + end + def with_schema(schema) schema_instance = schema.is_a?(Class) ? schema.new : schema @@ -121,7 +132,32 @@ def each(&) messages.each(&) end - def complete(&) # rubocop:disable Metrics/PerceivedComplexity + def complete(&) + span_name = "chat #{@model.id}" + Instrumentation.tracer(@config).in_span(span_name, kind: Instrumentation::SpanKind::CLIENT) do |span| + complete_with_span(span, &) + end + end + + def add_message(message_or_attributes) + message = message_or_attributes.is_a?(Message) ? message_or_attributes : Message.new(message_or_attributes) + messages << message + message + end + + def reset_messages! + @messages.clear + end + + def instance_variables + super - %i[@connection @config] + end + + private + + def complete_with_span(span, &) + add_request_span_attributes(span) + response = @provider.complete( messages, tools: @tools, @@ -133,6 +169,69 @@ def complete(&) # rubocop:disable Metrics/PerceivedComplexity &wrap_streaming_block(&) ) + add_response_span_attributes(span, response) + finalize_response(response, &) + rescue StandardError => e + record_span_error(span, e) + raise + end + + def add_request_span_attributes(span) + return unless span.recording? + + langsmith = @config.tracing_langsmith_compat + + span.add_attributes( + Instrumentation::SpanBuilder.build_request_attributes( + model: @model, + provider: @provider.slug, + session_id: @session_id, + config: { + temperature: @temperature, + metadata: @metadata, + langsmith_compat: langsmith, + metadata_prefix: @config.tracing_metadata_prefix + } + ) + ) + + return unless @config.tracing_log_content + + span.add_attributes( + Instrumentation::SpanBuilder.build_message_attributes( + messages, + max_length: @config.tracing_max_content_length, + langsmith_compat: langsmith + ) + ) + end + + def add_response_span_attributes(span, response) + return unless span.recording? + + span.add_attributes(Instrumentation::SpanBuilder.build_response_attributes(response)) + + return unless @config.tracing_log_content + + span.add_attributes( + Instrumentation::SpanBuilder.build_completion_attributes( + response, + max_length: @config.tracing_max_content_length, + langsmith_compat: @config.tracing_langsmith_compat + ) + ) + end + + def record_span_error(span, exception) + return unless span.recording? + + span.record_exception(exception) + return unless defined?(OpenTelemetry::Trace::Status) + + span.status = OpenTelemetry::Trace::Status.error(exception.message) + end + + def finalize_response(response, &) # rubocop:disable Metrics/PerceivedComplexity @on[:new_message]&.call unless block_given? if @schema && response.content.is_a?(String) @@ -153,22 +252,6 @@ def complete(&) # rubocop:disable Metrics/PerceivedComplexity end end - def add_message(message_or_attributes) - message = message_or_attributes.is_a?(Message) ? message_or_attributes : Message.new(message_or_attributes) - messages << message - message - end - - def reset_messages! - @messages.clear - end - - def instance_variables - super - %i[@connection @config] - end - - private - def wrap_streaming_block(&block) return nil unless block_given? @@ -205,9 +288,53 @@ def handle_tool_calls(response, &) # rubocop:disable Metrics/PerceivedComplexity end def execute_tool(tool_call) + span_name = "execute_tool #{tool_call.name}" + Instrumentation.tracer(@config).in_span(span_name, kind: Instrumentation::SpanKind::INTERNAL) do |span| + execute_tool_with_span(tool_call, span) + end + end + + def execute_tool_with_span(tool_call, span) tool = tools[tool_call.name.to_sym] args = tool_call.arguments - tool.call(args) + langsmith = @config.tracing_langsmith_compat + + if span.recording? + span.add_attributes( + Instrumentation::SpanBuilder.build_tool_attributes( + tool_call: tool_call, + session_id: @session_id, + langsmith_compat: langsmith + ) + ) + + if @config.tracing_log_content + span.add_attributes( + Instrumentation::SpanBuilder.build_tool_input_attributes( + tool_call: tool_call, + max_length: @config.tracing_max_content_length, + langsmith_compat: langsmith + ) + ) + end + end + + result = tool.call(args) + + if span.recording? && @config.tracing_log_content + span.add_attributes( + Instrumentation::SpanBuilder.build_tool_output_attributes( + result: result, + max_length: @config.tracing_max_content_length, + langsmith_compat: langsmith + ) + ) + end + + result + rescue StandardError => e + record_span_error(span, e) + raise end def build_content(message, attachments) diff --git a/lib/ruby_llm/configuration.rb b/lib/ruby_llm/configuration.rb index e1c12902a..482cb155d 100644 --- a/lib/ruby_llm/configuration.rb +++ b/lib/ruby_llm/configuration.rb @@ -46,7 +46,14 @@ class Configuration :logger, :log_file, :log_level, - :log_stream_debug + :log_stream_debug, + # Tracing configuration + :tracing_enabled, + :tracing_log_content, + :tracing_max_content_length, + :tracing_metadata_prefix + + attr_reader :tracing_langsmith_compat def initialize @request_timeout = 300 @@ -69,6 +76,24 @@ def initialize @log_file = $stdout @log_level = ENV['RUBYLLM_DEBUG'] ? Logger::DEBUG : Logger::INFO @log_stream_debug = ENV['RUBYLLM_STREAM_DEBUG'] == 'true' + + @tracing_enabled = false + @tracing_log_content = false + @tracing_max_content_length = 10_000 + @tracing_metadata_prefix = 'metadata' + @tracing_langsmith_compat = false + end + + def tracing_langsmith_compat=(value) + @tracing_langsmith_compat = value + if value + # Auto-set metadata prefix for LangSmith when enabling compat mode, + # but only if the user hasn't customized it + @tracing_metadata_prefix = 'langsmith.metadata' if @tracing_metadata_prefix == 'metadata' + elsif @tracing_metadata_prefix == 'langsmith.metadata' + # Revert to default when disabling compat mode (if still using langsmith prefix) + @tracing_metadata_prefix = 'metadata' + end end def instance_variables diff --git a/lib/ruby_llm/instrumentation.rb b/lib/ruby_llm/instrumentation.rb new file mode 100644 index 000000000..4ea490921 --- /dev/null +++ b/lib/ruby_llm/instrumentation.rb @@ -0,0 +1,249 @@ +# frozen_string_literal: true + +require 'json' +require 'singleton' + +module RubyLLM + # OpenTelemetry instrumentation for RubyLLM + # Provides tracing capabilities when enabled and OpenTelemetry is available + module Instrumentation + # Span kind constants (matches OpenTelemetry::Trace::SpanKind) + module SpanKind + CLIENT = :client + INTERNAL = :internal + end + + class << self + def enabled?(config = RubyLLM.config) + return false unless config.tracing_enabled + + unless otel_available? + warn_otel_missing unless @otel_warning_issued + @otel_warning_issued = true + return false + end + + true + end + + def tracer(config = RubyLLM.config) + return NullTracer.instance unless enabled?(config) + + # Don't memoize - tracer_provider can be reconfigured after initial load + OpenTelemetry.tracer_provider.tracer('ruby_llm', RubyLLM::VERSION) + end + + def reset! + @otel_warning_issued = false + end + + private + + def otel_available? + return false unless defined?(OpenTelemetry) + + !!OpenTelemetry.tracer_provider + end + + def warn_otel_missing + RubyLLM.logger.warn <<~MSG.strip + [RubyLLM] Tracing is enabled but OpenTelemetry is not available. + Tracing will be disabled. To enable, add to your Gemfile: + + gem 'opentelemetry-sdk' + gem 'opentelemetry-exporter-otlp' + + See https://rubyllm.com/advanced/observability for setup instructions. + MSG + end + end + + # No-op tracer used when tracing is disabled or OpenTelemetry is not available + class NullTracer + include Singleton + + def in_span(_name, **_options) + yield NullSpan.instance + end + end + + # No-op span that responds to all span methods but does nothing + class NullSpan + include Singleton + + def recording? + false + end + + def set_attribute(_key, _value) + self + end + + def add_attributes(_attributes) + self + end + + def record_exception(_exception, _attributes = {}) + self + end + + def status=(_status) + # no-op + end + + def finish + self + end + end + + # Helper for building span attributes + module SpanBuilder + class << self + def truncate_content(content, max_length) + return nil if content.nil? + + content_str = content.to_s + return content_str if content_str.length <= max_length + + "#{content_str[0, max_length]}... [truncated]" + end + + def extract_content_text(content) + case content + when String + content + when RubyLLM::Content + content.text || describe_attachments(content.attachments) + else + content.to_s + end + end + + def describe_attachments(attachments) + return '[no content]' if attachments.empty? + + descriptions = attachments.map { |a| "#{a.type}: #{a.filename}" } + "[#{descriptions.join(', ')}]" + end + + def build_message_attributes(messages, max_length:, langsmith_compat: false) + attrs = {} + + # Build OTEL GenAI spec-compliant input messages + input_messages = messages.map do |msg| + build_message_object(msg) + end + attrs['gen_ai.input.messages'] = truncate_content(JSON.generate(input_messages), max_length) + + # Set input.value for LangSmith Input panel (last user message) + if langsmith_compat + last_user_msg = messages.reverse.find { |m| m.role.to_s == 'user' } + if last_user_msg + content = extract_content_text(last_user_msg.content) + attrs['input.value'] = truncate_content(content, max_length) + end + end + attrs + end + + def build_completion_attributes(message, max_length:, langsmith_compat: false) + attrs = {} + + # Build OTEL GenAI spec-compliant output messages + output_messages = [build_message_object(message)] + attrs['gen_ai.output.messages'] = truncate_content(JSON.generate(output_messages), max_length) + + # Set output.value for LangSmith Output panel + if langsmith_compat + content = extract_content_text(message.content) + attrs['output.value'] = truncate_content(content, max_length) + end + attrs + end + + def build_message_object(message) + content = extract_content_text(message.content) + obj = { + role: message.role.to_s, + parts: [{ type: 'text', content: content }] + } + + # Add tool calls if present + if message.respond_to?(:tool_calls) && message.tool_calls&.any? + message.tool_calls.each_value do |tc| + obj[:parts] << { + type: 'tool_call', + id: tc.id, + name: tc.name, + arguments: tc.arguments + } + end + end + + obj + end + + def build_request_attributes(model:, provider:, session_id:, config: {}) + attrs = { + 'gen_ai.provider.name' => provider.to_s, + 'gen_ai.operation.name' => 'chat', + 'gen_ai.request.model' => model.id, + 'gen_ai.conversation.id' => session_id + } + attrs['langsmith.span.kind'] = 'LLM' if config[:langsmith_compat] + attrs['gen_ai.request.temperature'] = config[:temperature] if config[:temperature] + if config[:metadata]&.any? + build_metadata_attributes(attrs, config[:metadata], + prefix: config[:metadata_prefix]) + end + attrs + end + + def build_metadata_attributes(attrs, metadata, prefix: 'metadata') + metadata.each do |key, value| + next if value.nil? + + # Let OTel SDK handle type coercion for supported types + attrs["#{prefix}.#{key}"] = value + end + end + + def build_response_attributes(response) + attrs = {} + attrs['gen_ai.response.model'] = response.model_id if response.model_id + attrs['gen_ai.usage.input_tokens'] = response.input_tokens if response.input_tokens + attrs['gen_ai.usage.output_tokens'] = response.output_tokens if response.output_tokens + attrs + end + + def build_tool_attributes(tool_call:, session_id:, langsmith_compat: false) + attrs = { + 'gen_ai.operation.name' => 'execute_tool', + 'gen_ai.tool.name' => tool_call.name.to_s, + 'gen_ai.tool.call.id' => tool_call.id, + 'gen_ai.conversation.id' => session_id + } + attrs['langsmith.span.kind'] = 'TOOL' if langsmith_compat + attrs + end + + def build_tool_input_attributes(tool_call:, max_length:, langsmith_compat: false) + args = tool_call.arguments + input = args.is_a?(String) ? args : JSON.generate(args) + truncated = truncate_content(input, max_length) + attrs = { 'gen_ai.tool.call.arguments' => truncated } + attrs['input.value'] = truncated if langsmith_compat + attrs + end + + def build_tool_output_attributes(result:, max_length:, langsmith_compat: false) + output = result.is_a?(String) ? result : result.to_s + truncated = truncate_content(output, max_length) + attrs = { 'gen_ai.tool.call.result' => truncated } + attrs['output.value'] = truncated if langsmith_compat + attrs + end + end + end + end +end diff --git a/lib/ruby_llm/tool.rb b/lib/ruby_llm/tool.rb index 2846a995d..24d2468c7 100644 --- a/lib/ruby_llm/tool.rb +++ b/lib/ruby_llm/tool.rb @@ -186,7 +186,7 @@ def resolve_schema def resolve_direct_schema(schema) return extract_schema(schema.to_json_schema) if schema.respond_to?(:to_json_schema) return RubyLLM::Utils.deep_dup(schema) if schema.is_a?(Hash) - if schema.is_a?(Class) && schema.instance_methods.include?(:to_json_schema) + if schema.is_a?(Class) && schema.method_defined?(:to_json_schema) return extract_schema(schema.new.to_json_schema) end diff --git a/spec/fixtures/vcr_cassettes/instrumentation_creates_spans_for_chat.yml b/spec/fixtures/vcr_cassettes/instrumentation_creates_spans_for_chat.yml new file mode 100644 index 000000000..89eb65215 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/instrumentation_creates_spans_for_chat.yml @@ -0,0 +1,116 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Hello"}],"stream":false}' + headers: + User-Agent: + - Faraday v2.14.0 + Authorization: + - Bearer + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Fri, 05 Dec 2025 20:43:10 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - "" + Openai-Processing-Ms: + - '210' + Openai-Project: + - proj_NKne139JTtyy43xNywVdTYx1 + Openai-Version: + - '2020-10-01' + X-Envoy-Upstream-Service-Time: + - '366' + X-Ratelimit-Limit-Requests: + - '30000' + X-Ratelimit-Limit-Tokens: + - '150000000' + X-Ratelimit-Remaining-Requests: + - '29999' + X-Ratelimit-Remaining-Tokens: + - '149999995' + X-Ratelimit-Reset-Requests: + - 2ms + X-Ratelimit-Reset-Tokens: + - 0s + X-Request-Id: + - "" + X-Openai-Proxy-Wasm: + - v0.1 + Cf-Cache-Status: + - DYNAMIC + Set-Cookie: + - "" + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + Server: + - cloudflare + Cf-Ray: + - "" + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: | + { + "id": "chatcmpl-CjWvuw5taodyXNSbbwC53IudDDQVq", + "object": "chat.completion", + "created": 1764967390, + "model": "gpt-4.1-nano-2025-04-14", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 8, + "completion_tokens": 9, + "total_tokens": 17, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_ef015fa747" + } + recorded_at: Fri, 05 Dec 2025 20:43:10 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/instrumentation_creates_tool_spans.yml b/spec/fixtures/vcr_cassettes/instrumentation_creates_tool_spans.yml new file mode 100644 index 000000000..50803c053 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/instrumentation_creates_tool_spans.yml @@ -0,0 +1,213 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"}],"stream":false,"tools":[{"type":"function","function":{"name":"weather","description":"Gets + current weather for a location","parameters":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"],"additionalProperties":false,"strict":true}}}]}' + headers: + User-Agent: + - Faraday v2.14.0 + Authorization: + - Bearer + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Fri, 05 Dec 2025 20:43:11 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - "" + Openai-Processing-Ms: + - '623' + Openai-Project: + - proj_NKne139JTtyy43xNywVdTYx1 + Openai-Version: + - '2020-10-01' + X-Envoy-Upstream-Service-Time: + - '711' + X-Ratelimit-Limit-Requests: + - '30000' + X-Ratelimit-Limit-Tokens: + - '150000000' + X-Ratelimit-Remaining-Requests: + - '29999' + X-Ratelimit-Remaining-Tokens: + - '149999985' + X-Ratelimit-Reset-Requests: + - 2ms + X-Ratelimit-Reset-Tokens: + - 0s + X-Request-Id: + - "" + X-Openai-Proxy-Wasm: + - v0.1 + Cf-Cache-Status: + - DYNAMIC + Set-Cookie: + - "" + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + Server: + - cloudflare + Cf-Ray: + - "" + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: | + { + "id": "chatcmpl-CjWvvIyt3nRejD0oyQIfdc93V9WvT", + "object": "chat.completion", + "created": 1764967391, + "model": "gpt-4.1-nano-2025-04-14", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_adL6PJ7P2FJ8Aci4Mg0c4HIp", + "type": "function", + "function": { + "name": "weather", + "arguments": "{\"latitude\": \"52.5200\", \"longitude\": \"13.4050\"}" + } + } + ], + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 88, + "completion_tokens": 39, + "total_tokens": 127, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_1a97b5aa6c" + } + recorded_at: Fri, 05 Dec 2025 20:43:11 GMT +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","tool_calls":[{"id":"call_adL6PJ7P2FJ8Aci4Mg0c4HIp","type":"function","function":{"name":"weather","arguments":"{\"latitude\":\"52.5200\",\"longitude\":\"13.4050\"}"}}]},{"role":"tool","content":"Current + weather at 52.5200, 13.4050: 15°C, Wind: 10 km/h","tool_call_id":"call_adL6PJ7P2FJ8Aci4Mg0c4HIp"}],"stream":false,"tools":[{"type":"function","function":{"name":"weather","description":"Gets + current weather for a location","parameters":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"],"additionalProperties":false,"strict":true}}}]}' + headers: + User-Agent: + - Faraday v2.14.0 + Authorization: + - Bearer + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Fri, 05 Dec 2025 20:43:12 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - "" + Openai-Processing-Ms: + - '425' + Openai-Project: + - proj_NKne139JTtyy43xNywVdTYx1 + Openai-Version: + - '2020-10-01' + X-Envoy-Upstream-Service-Time: + - '532' + X-Ratelimit-Limit-Requests: + - '30000' + X-Ratelimit-Limit-Tokens: + - '150000000' + X-Ratelimit-Remaining-Requests: + - '29999' + X-Ratelimit-Remaining-Tokens: + - '149999967' + X-Ratelimit-Reset-Requests: + - 2ms + X-Ratelimit-Reset-Tokens: + - 0s + X-Request-Id: + - "" + X-Openai-Proxy-Wasm: + - v0.1 + Cf-Cache-Status: + - DYNAMIC + Set-Cookie: + - "" + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + Server: + - cloudflare + Cf-Ray: + - "" + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + ewogICJpZCI6ICJjaGF0Y21wbC1Dald2d0Z6NHFKMjNGNlFKT24zanlkWXFOaTRRQiIsCiAgIm9iamVjdCI6ICJjaGF0LmNvbXBsZXRpb24iLAogICJjcmVhdGVkIjogMTc2NDk2NzM5MiwKICAibW9kZWwiOiAiZ3B0LTQuMS1uYW5vLTIwMjUtMDQtMTQiLAogICJjaG9pY2VzIjogWwogICAgewogICAgICAiaW5kZXgiOiAwLAogICAgICAibWVzc2FnZSI6IHsKICAgICAgICAicm9sZSI6ICJhc3Npc3RhbnQiLAogICAgICAgICJjb250ZW50IjogIlRoZSBjdXJyZW50IHdlYXRoZXIgaW4gQmVybGluIGlzIDE1wrBDIHdpdGggYSB3aW5kIHNwZWVkIG9mIDEwIGttL2guIiwKICAgICAgICAicmVmdXNhbCI6IG51bGwsCiAgICAgICAgImFubm90YXRpb25zIjogW10KICAgICAgfSwKICAgICAgImxvZ3Byb2JzIjogbnVsbCwKICAgICAgImZpbmlzaF9yZWFzb24iOiAic3RvcCIKICAgIH0KICBdLAogICJ1c2FnZSI6IHsKICAgICJwcm9tcHRfdG9rZW5zIjogMTQzLAogICAgImNvbXBsZXRpb25fdG9rZW5zIjogMjAsCiAgICAidG90YWxfdG9rZW5zIjogMTYzLAogICAgInByb21wdF90b2tlbnNfZGV0YWlscyI6IHsKICAgICAgImNhY2hlZF90b2tlbnMiOiAwLAogICAgICAiYXVkaW9fdG9rZW5zIjogMAogICAgfSwKICAgICJjb21wbGV0aW9uX3Rva2Vuc19kZXRhaWxzIjogewogICAgICAicmVhc29uaW5nX3Rva2VucyI6IDAsCiAgICAgICJhdWRpb190b2tlbnMiOiAwLAogICAgICAiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOiAwLAogICAgICAicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOiAwCiAgICB9CiAgfSwKICAic2VydmljZV90aWVyIjogImRlZmF1bHQiLAogICJzeXN0ZW1fZmluZ2VycHJpbnQiOiAiZnBfMWE5N2I1YWE2YyIKfQo= + recorded_at: Fri, 05 Dec 2025 20:43:12 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/instrumentation_includes_token_usage.yml b/spec/fixtures/vcr_cassettes/instrumentation_includes_token_usage.yml new file mode 100644 index 000000000..3017b4813 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/instrumentation_includes_token_usage.yml @@ -0,0 +1,116 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"Hello"}],"stream":false}' + headers: + User-Agent: + - Faraday v2.14.0 + Authorization: + - Bearer + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Fri, 05 Dec 2025 20:43:13 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - "" + Openai-Processing-Ms: + - '272' + Openai-Project: + - proj_NKne139JTtyy43xNywVdTYx1 + Openai-Version: + - '2020-10-01' + X-Envoy-Upstream-Service-Time: + - '375' + X-Ratelimit-Limit-Requests: + - '30000' + X-Ratelimit-Limit-Tokens: + - '150000000' + X-Ratelimit-Remaining-Requests: + - '29999' + X-Ratelimit-Remaining-Tokens: + - '149999995' + X-Ratelimit-Reset-Requests: + - 2ms + X-Ratelimit-Reset-Tokens: + - 0s + X-Request-Id: + - "" + X-Openai-Proxy-Wasm: + - v0.1 + Cf-Cache-Status: + - DYNAMIC + Set-Cookie: + - "" + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + Server: + - cloudflare + Cf-Ray: + - "" + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: | + { + "id": "chatcmpl-CjWvxT11aJ9cmSH46FoIZ9X4wrKbb", + "object": "chat.completion", + "created": 1764967393, + "model": "gpt-4.1-nano-2025-04-14", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 8, + "completion_tokens": 9, + "total_tokens": 17, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_ef015fa747" + } + recorded_at: Fri, 05 Dec 2025 20:43:13 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/instrumentation_maintains_session_id.yml b/spec/fixtures/vcr_cassettes/instrumentation_maintains_session_id.yml new file mode 100644 index 000000000..fa18f2624 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/instrumentation_maintains_session_id.yml @@ -0,0 +1,198 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"What''s + your favorite color?"}],"stream":false}' + headers: + User-Agent: + - Faraday v2.14.0 + Authorization: + - Bearer + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Fri, 05 Dec 2025 20:43:14 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - "" + Openai-Processing-Ms: + - '521' + Openai-Project: + - proj_NKne139JTtyy43xNywVdTYx1 + Openai-Version: + - '2020-10-01' + X-Envoy-Upstream-Service-Time: + - '613' + X-Ratelimit-Limit-Requests: + - '30000' + X-Ratelimit-Limit-Tokens: + - '150000000' + X-Ratelimit-Remaining-Requests: + - '29999' + X-Ratelimit-Remaining-Tokens: + - '149999990' + X-Ratelimit-Reset-Requests: + - 2ms + X-Ratelimit-Reset-Tokens: + - 0s + X-Request-Id: + - "" + X-Openai-Proxy-Wasm: + - v0.1 + Cf-Cache-Status: + - DYNAMIC + Set-Cookie: + - "" + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + Server: + - cloudflare + Cf-Ray: + - "" + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: | + { + "id": "chatcmpl-CjWvxIL5R4NWCVmGoornkRReaUDZH", + "object": "chat.completion", + "created": 1764967393, + "model": "gpt-4.1-nano-2025-04-14", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "I don't have personal feelings or preferences, but I think the color blue is quite popular and calming! Do you have a favorite color?", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 27, + "total_tokens": 39, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_ef015fa747" + } + recorded_at: Fri, 05 Dec 2025 20:43:14 GMT +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"gpt-4.1-nano","messages":[{"role":"user","content":"What''s + your favorite color?"},{"role":"assistant","content":"I don''t have personal + feelings or preferences, but I think the color blue is quite popular and calming! + Do you have a favorite color?"},{"role":"user","content":"Why is that?"}],"stream":false}' + headers: + User-Agent: + - Faraday v2.14.0 + Authorization: + - Bearer + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Fri, 05 Dec 2025 20:43:15 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - "" + Openai-Processing-Ms: + - '855' + Openai-Project: + - proj_NKne139JTtyy43xNywVdTYx1 + Openai-Version: + - '2020-10-01' + X-Envoy-Upstream-Service-Time: + - '995' + X-Ratelimit-Limit-Requests: + - '30000' + X-Ratelimit-Limit-Tokens: + - '150000000' + X-Ratelimit-Remaining-Requests: + - '29999' + X-Ratelimit-Remaining-Tokens: + - '149999952' + X-Ratelimit-Reset-Requests: + - 2ms + X-Ratelimit-Reset-Tokens: + - 0s + X-Request-Id: + - "" + X-Openai-Proxy-Wasm: + - v0.1 + Cf-Cache-Status: + - DYNAMIC + Set-Cookie: + - "" + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + Server: + - cloudflare + Cf-Ray: + - "" + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: !binary |- + ewogICJpZCI6ICJjaGF0Y21wbC1Dald2emZnM0dqaWxsdnpId3U1R3VDZnZKeXBTMCIsCiAgIm9iamVjdCI6ICJjaGF0LmNvbXBsZXRpb24iLAogICJjcmVhdGVkIjogMTc2NDk2NzM5NSwKICAibW9kZWwiOiAiZ3B0LTQuMS1uYW5vLTIwMjUtMDQtMTQiLAogICJjaG9pY2VzIjogWwogICAgewogICAgICAiaW5kZXgiOiAwLAogICAgICAibWVzc2FnZSI6IHsKICAgICAgICAicm9sZSI6ICJhc3Npc3RhbnQiLAogICAgICAgICJjb250ZW50IjogIlBlb3BsZSBvZnRlbiBmYXZvciBibHVlIGJlY2F1c2UgaXQncyBhc3NvY2lhdGVkIHdpdGggcXVhbGl0aWVzIGxpa2UgY2FsbW5lc3MsIHRydXN0LCBhbmQgdHJhbnF1aWxpdHkuIEl0J3MgYSBjb2xvciBjb21tb25seSBmb3VuZCBpbiBuYXR1cmXigJR0aGluayB0aGUgc2t5IGFuZCB0aGUgb2NlYW7igJR3aGljaCBjYW4gZXZva2UgZmVlbGluZ3Mgb2YgcGVhY2UgYW5kIHJlbGF4YXRpb24uIEFkZGl0aW9uYWxseSwgbWFueSBjdWx0dXJlcyBzZWUgYmx1ZSBhcyBhIHN0YWJsZSBhbmQgZGVwZW5kYWJsZSBjb2xvciwgd2hpY2ggbWlnaHQgY29udHJpYnV0ZSB0byBpdHMgcG9wdWxhcml0eS4gRG8geW91IGhhdmUgYSByZWFzb24gd2h5IHlvdSBsaWtlIGEgcGFydGljdWxhciBjb2xvcj8iLAogICAgICAgICJyZWZ1c2FsIjogbnVsbCwKICAgICAgICAiYW5ub3RhdGlvbnMiOiBbXQogICAgICB9LAogICAgICAibG9ncHJvYnMiOiBudWxsLAogICAgICAiZmluaXNoX3JlYXNvbiI6ICJzdG9wIgogICAgfQogIF0sCiAgInVzYWdlIjogewogICAgInByb21wdF90b2tlbnMiOiA1MSwKICAgICJjb21wbGV0aW9uX3Rva2VucyI6IDczLAogICAgInRvdGFsX3Rva2VucyI6IDEyNCwKICAgICJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOiB7CiAgICAgICJjYWNoZWRfdG9rZW5zIjogMCwKICAgICAgImF1ZGlvX3Rva2VucyI6IDAKICAgIH0sCiAgICAiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6IHsKICAgICAgInJlYXNvbmluZ190b2tlbnMiOiAwLAogICAgICAiYXVkaW9fdG9rZW5zIjogMCwKICAgICAgImFjY2VwdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjogMCwKICAgICAgInJlamVjdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjogMAogICAgfQogIH0sCiAgInNlcnZpY2VfdGllciI6ICJkZWZhdWx0IiwKICAic3lzdGVtX2ZpbmdlcnByaW50IjogImZwX2VmMDE1ZmE3NDciCn0K + recorded_at: Fri, 05 Dec 2025 20:43:15 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/ruby_llm/active_record/acts_as_model_spec.rb b/spec/ruby_llm/active_record/acts_as_model_spec.rb index 56fb01570..e72c6731d 100644 --- a/spec/ruby_llm/active_record/acts_as_model_spec.rb +++ b/spec/ruby_llm/active_record/acts_as_model_spec.rb @@ -255,7 +255,8 @@ def messages # Mock the chat creation to verify parameters expect(RubyLLM).to receive(:chat).with( # rubocop:disable RSpec/MessageSpies,RSpec/StubbedMock model: 'test-gpt', - provider: :openai + provider: :openai, + session_id: "#{chat.class.name}:#{chat.id}" ).and_return( instance_double(RubyLLM::Chat, reset_messages!: nil, add_message: nil, instance_variable_get: {}, on_new_message: nil, on_end_message: nil, @@ -272,7 +273,8 @@ def messages expect(RubyLLM).to receive(:chat).with( # rubocop:disable RSpec/MessageSpies,RSpec/StubbedMock model: 'test-claude', - provider: :anthropic + provider: :anthropic, + session_id: "#{chat.class.name}:#{chat.id}" ).and_return( instance_double(RubyLLM::Chat, reset_messages!: nil, add_message: nil, instance_variable_get: {}, on_new_message: nil, on_end_message: nil, diff --git a/spec/ruby_llm/instrumentation_spec.rb b/spec/ruby_llm/instrumentation_spec.rb new file mode 100644 index 000000000..8cec220a4 --- /dev/null +++ b/spec/ruby_llm/instrumentation_spec.rb @@ -0,0 +1,863 @@ +# frozen_string_literal: true + +require 'opentelemetry/sdk' + +RSpec.describe RubyLLM::Instrumentation do + describe 'Configuration' do + it 'has tracing_enabled defaulting to false' do + config = RubyLLM::Configuration.new + expect(config.tracing_enabled).to be false + end + + it 'has tracing_log_content defaulting to false' do + config = RubyLLM::Configuration.new + expect(config.tracing_log_content).to be false + end + + it 'has tracing_max_content_length defaulting to 10000' do + config = RubyLLM::Configuration.new + expect(config.tracing_max_content_length).to eq 10_000 + end + + it 'has tracing_metadata_prefix defaulting to metadata' do + config = RubyLLM::Configuration.new + expect(config.tracing_metadata_prefix).to eq 'metadata' + end + + it 'has tracing_langsmith_compat defaulting to false' do + config = RubyLLM::Configuration.new + expect(config.tracing_langsmith_compat).to be false + end + + it 'auto-sets tracing_metadata_prefix when enabling langsmith_compat' do + config = RubyLLM::Configuration.new + expect(config.tracing_metadata_prefix).to eq 'metadata' + + config.tracing_langsmith_compat = true + expect(config.tracing_metadata_prefix).to eq 'langsmith.metadata' + end + + it 'does not override custom tracing_metadata_prefix when enabling langsmith_compat' do + config = RubyLLM::Configuration.new + config.tracing_metadata_prefix = 'app.custom' + + config.tracing_langsmith_compat = true + expect(config.tracing_metadata_prefix).to eq 'app.custom' + end + + it 'reverts tracing_metadata_prefix when disabling langsmith_compat' do + config = RubyLLM::Configuration.new + config.tracing_langsmith_compat = true + expect(config.tracing_metadata_prefix).to eq 'langsmith.metadata' + + config.tracing_langsmith_compat = false + expect(config.tracing_metadata_prefix).to eq 'metadata' + end + + it 'does not revert custom prefix when disabling langsmith_compat' do + config = RubyLLM::Configuration.new + config.tracing_langsmith_compat = true + config.tracing_metadata_prefix = 'app.custom' + + config.tracing_langsmith_compat = false + expect(config.tracing_metadata_prefix).to eq 'app.custom' + end + + it 'allows configuration via block' do + RubyLLM.configure do |config| + config.tracing_enabled = true + config.tracing_log_content = true + config.tracing_max_content_length = 5000 + config.tracing_metadata_prefix = 'app.metadata' + end + + expect(RubyLLM.config.tracing_enabled).to be true + expect(RubyLLM.config.tracing_log_content).to be true + expect(RubyLLM.config.tracing_max_content_length).to eq 5000 + expect(RubyLLM.config.tracing_metadata_prefix).to eq 'app.metadata' + end + end + + describe 'RubyLLM::Instrumentation' do + describe '.enabled?' do + it 'returns false when tracing_enabled is false' do + RubyLLM.configure { |c| c.tracing_enabled = false } + expect(described_class.enabled?).to be false + end + + it 'returns true when tracing_enabled is true and OpenTelemetry is available' do + RubyLLM.configure { |c| c.tracing_enabled = true } + described_class.reset! + expect(described_class.enabled?).to be true + end + + it 'returns false and warns when tracing_enabled is true but OpenTelemetry is not available' do + RubyLLM.configure { |c| c.tracing_enabled = true } + described_class.reset! + allow(described_class).to receive(:otel_available?).and_return(false) + allow(RubyLLM.logger).to receive(:warn) + + expect(described_class.enabled?).to be false + expect(RubyLLM.logger).to have_received(:warn).with(/OpenTelemetry is not available/) + end + + it 'only warns once per reset cycle' do + RubyLLM.configure { |c| c.tracing_enabled = true } + described_class.reset! + allow(described_class).to receive(:otel_available?).and_return(false) + allow(RubyLLM.logger).to receive(:warn) + + described_class.enabled? + described_class.enabled? + described_class.enabled? + + expect(RubyLLM.logger).to have_received(:warn).once + end + end + + describe '.tracer' do + it 'returns NullTracer when disabled' do + RubyLLM.configure { |c| c.tracing_enabled = false } + expect(described_class.tracer).to be_a(RubyLLM::Instrumentation::NullTracer) + end + end + end + + describe 'RubyLLM::Instrumentation::NullTracer' do + let(:tracer) { RubyLLM::Instrumentation::NullTracer.instance } + + it 'is a singleton' do + expect(tracer).to be RubyLLM::Instrumentation::NullTracer.instance + end + + describe '#in_span' do + it 'yields a NullSpan' do + tracer.in_span('test.span') do |span| + expect(span).to be_a(RubyLLM::Instrumentation::NullSpan) + end + end + + it 'returns the block result' do + result = tracer.in_span('test.span') { 'hello' } + expect(result).to eq 'hello' + end + + it 'propagates exceptions' do + expect do + tracer.in_span('test.span') { raise 'boom' } + end.to raise_error('boom') + end + end + end + + describe 'RubyLLM::Instrumentation::NullSpan' do + let(:span) { RubyLLM::Instrumentation::NullSpan.instance } + + it 'is a singleton' do + expect(span).to be RubyLLM::Instrumentation::NullSpan.instance + end + + it 'responds to recording? and returns false' do + expect(span.recording?).to be false + end + + it 'responds to set_attribute and returns self' do + expect(span.set_attribute('key', 'value')).to be span + end + + it 'responds to add_attributes and returns self' do + expect(span.add_attributes(key: 'value')).to be span + end + + it 'responds to record_exception and returns self' do + expect(span.record_exception(StandardError.new)).to be span + end + + it 'responds to status= and returns nil' do + expect(span.status = 'error').to eq 'error' + end + end + + describe 'Chat#session_id' do + include_context 'with configured RubyLLM' + + it 'generates a unique session_id for each Chat instance' do + chat1 = RubyLLM::Chat.new(model: 'gpt-4o-mini', assume_model_exists: true, provider: :openai) + chat2 = RubyLLM::Chat.new(model: 'gpt-4o-mini', assume_model_exists: true, provider: :openai) + + expect(chat1.session_id).to be_a(String) + expect(chat1.session_id).to match(/\A[0-9a-f-]{36}\z/) # UUID format + expect(chat1.session_id).not_to eq(chat2.session_id) + end + + it 'maintains the same session_id across multiple asks' do + chat = RubyLLM::Chat.new(model: 'gpt-4o-mini', assume_model_exists: true, provider: :openai) + session_id = chat.session_id + + # session_id should remain constant + expect(chat.session_id).to eq(session_id) + expect(chat.session_id).to eq(session_id) + end + + it 'accepts a custom session_id' do + custom_id = 'my-conversation-123' + chat = RubyLLM::Chat.new(model: 'gpt-4o-mini', assume_model_exists: true, provider: :openai, + session_id: custom_id) + + expect(chat.session_id).to eq(custom_id) + end + + it 'generates a UUID when session_id is nil' do + chat = RubyLLM::Chat.new(model: 'gpt-4o-mini', assume_model_exists: true, provider: :openai, session_id: nil) + + expect(chat.session_id).to be_a(String) + expect(chat.session_id).to match(/\A[0-9a-f-]{36}\z/) + end + end + + describe 'RubyLLM::Instrumentation::SpanBuilder' do + describe '.build_tool_attributes' do + it 'builds attributes for a tool call' do + tool_call = instance_double(RubyLLM::ToolCall, name: 'get_weather', id: 'call_123', + arguments: { location: 'NYC' }) + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_tool_attributes( + tool_call: tool_call, + session_id: 'session-abc' + ) + + expect(attrs['gen_ai.operation.name']).to eq 'execute_tool' + expect(attrs['gen_ai.tool.name']).to eq 'get_weather' + expect(attrs['gen_ai.tool.call.id']).to eq 'call_123' + expect(attrs['gen_ai.conversation.id']).to eq 'session-abc' + expect(attrs).not_to have_key('langsmith.span.kind') + end + + it 'includes langsmith.span.kind when langsmith_compat is true' do + tool_call = instance_double(RubyLLM::ToolCall, name: 'get_weather', id: 'call_123', + arguments: { location: 'NYC' }) + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_tool_attributes( + tool_call: tool_call, + session_id: 'session-abc', + langsmith_compat: true + ) + + expect(attrs['langsmith.span.kind']).to eq 'TOOL' + end + end + + describe '.build_tool_input_attributes' do + it 'builds input attributes with truncation' do + tool_call = instance_double(RubyLLM::ToolCall, arguments: { query: 'a' * 100 }) + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_tool_input_attributes( + tool_call: tool_call, + max_length: 50 + ) + + expect(attrs['gen_ai.tool.call.arguments']).to include('[truncated]') + expect(attrs).not_to have_key('input.value') + end + + it 'includes input.value when langsmith_compat is true' do + tool_call = instance_double(RubyLLM::ToolCall, arguments: { query: 'test' }) + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_tool_input_attributes( + tool_call: tool_call, + max_length: 100, + langsmith_compat: true + ) + + expect(attrs['gen_ai.tool.call.arguments']).to be_a(String) + expect(attrs['input.value']).to eq(attrs['gen_ai.tool.call.arguments']) + end + end + + describe '.build_tool_output_attributes' do + it 'builds output attributes with truncation' do + result = 'b' * 100 + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_tool_output_attributes( + result: result, + max_length: 50 + ) + + expect(attrs['gen_ai.tool.call.result']).to include('[truncated]') + expect(attrs).not_to have_key('output.value') + end + + it 'includes output.value when langsmith_compat is true' do + result = 'test output' + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_tool_output_attributes( + result: result, + max_length: 100, + langsmith_compat: true + ) + + expect(attrs['gen_ai.tool.call.result']).to eq('test output') + expect(attrs['output.value']).to eq('test output') + end + end + + describe '.truncate_content' do + it 'returns content as-is when under max length' do + content = 'short content' + result = RubyLLM::Instrumentation::SpanBuilder.truncate_content(content, 100) + expect(result).to eq 'short content' + end + + it 'truncates content when over max length' do + content = 'a' * 100 + result = RubyLLM::Instrumentation::SpanBuilder.truncate_content(content, 50) + expect(result).to eq "#{'a' * 50}... [truncated]" + end + + it 'handles nil content' do + result = RubyLLM::Instrumentation::SpanBuilder.truncate_content(nil, 100) + expect(result).to be_nil + end + end + + describe '.extract_content_text' do + it 'returns string content as-is' do + result = RubyLLM::Instrumentation::SpanBuilder.extract_content_text('Hello') + expect(result).to eq 'Hello' + end + + it 'extracts text from Content objects' do + content = RubyLLM::Content.new('Hello with attachment', ['spec/fixtures/ruby.png']) + result = RubyLLM::Instrumentation::SpanBuilder.extract_content_text(content) + expect(result).to eq 'Hello with attachment' + end + + it 'describes attachments for Content objects without text' do + content = RubyLLM::Content.new(nil, ['spec/fixtures/ruby.png']) + result = RubyLLM::Instrumentation::SpanBuilder.extract_content_text(content) + expect(result).to eq '[image: ruby.png]' + end + + it 'describes multiple attachments' do + content = RubyLLM::Content.new(nil, ['spec/fixtures/ruby.png', 'spec/fixtures/ruby.mp3']) + result = RubyLLM::Instrumentation::SpanBuilder.extract_content_text(content) + expect(result).to eq '[image: ruby.png, audio: ruby.mp3]' + end + end + + describe '.build_message_attributes' do + it 'builds gen_ai.input.messages as JSON array' do + messages = [ + RubyLLM::Message.new(role: :system, content: 'You are helpful'), + RubyLLM::Message.new(role: :user, content: 'Hello') + ] + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_message_attributes(messages, max_length: 10_000) + + parsed = JSON.parse(attrs['gen_ai.input.messages']) + expect(parsed).to be_an(Array) + expect(parsed.length).to eq 2 + expect(parsed[0]['role']).to eq 'system' + expect(parsed[0]['parts'][0]['content']).to eq 'You are helpful' + expect(parsed[1]['role']).to eq 'user' + expect(parsed[1]['parts'][0]['content']).to eq 'Hello' + end + + it 'truncates long JSON content' do + messages = [RubyLLM::Message.new(role: :user, content: 'a' * 100)] + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_message_attributes(messages, max_length: 50) + + expect(attrs['gen_ai.input.messages']).to include('[truncated]') + expect(attrs['gen_ai.input.messages'].length).to be <= 70 # 50 + "... [truncated]" + end + + it 'handles Content objects with attachments' do + content = RubyLLM::Content.new('Describe this image', ['spec/fixtures/ruby.png']) + messages = [RubyLLM::Message.new(role: :user, content: content)] + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_message_attributes(messages, max_length: 10_000) + + parsed = JSON.parse(attrs['gen_ai.input.messages']) + expect(parsed[0]['parts'][0]['content']).to eq 'Describe this image' + end + + it 'does not include input.value by default' do + messages = [RubyLLM::Message.new(role: :user, content: 'Hello')] + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_message_attributes(messages, max_length: 10_000) + + expect(attrs).not_to have_key('input.value') + end + + it 'includes input.value when langsmith_compat is true' do + messages = [RubyLLM::Message.new(role: :user, content: 'Hello')] + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_message_attributes( + messages, + max_length: 10_000, + langsmith_compat: true + ) + + expect(attrs['input.value']).to eq 'Hello' + end + end + + describe '.build_completion_attributes' do + it 'builds gen_ai.output.messages as JSON array' do + message = RubyLLM::Message.new(role: :assistant, content: 'Hi there!') + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_completion_attributes(message, max_length: 10_000) + + parsed = JSON.parse(attrs['gen_ai.output.messages']) + expect(parsed).to be_an(Array) + expect(parsed.length).to eq 1 + expect(parsed[0]['role']).to eq 'assistant' + expect(parsed[0]['parts'][0]['content']).to eq 'Hi there!' + expect(attrs).not_to have_key('output.value') + end + + it 'includes output.value when langsmith_compat is true' do + message = RubyLLM::Message.new(role: :assistant, content: 'Hi there!') + + attrs = RubyLLM::Instrumentation::SpanBuilder.build_completion_attributes( + message, + max_length: 10_000, + langsmith_compat: true + ) + + expect(attrs['output.value']).to eq 'Hi there!' + end + end + + describe '.build_request_attributes' do + let(:model) { instance_double(RubyLLM::Model::Info, id: 'gpt-4') } + + it 'does not include langsmith.span.kind by default' do + attrs = RubyLLM::Instrumentation::SpanBuilder.build_request_attributes( + model: model, + provider: :openai, + session_id: 'session-123' + ) + + expect(attrs['gen_ai.provider.name']).to eq 'openai' + expect(attrs['gen_ai.request.model']).to eq 'gpt-4' + expect(attrs).not_to have_key('langsmith.span.kind') + end + + it 'includes langsmith.span.kind when langsmith_compat is true' do + attrs = RubyLLM::Instrumentation::SpanBuilder.build_request_attributes( + model: model, + provider: :openai, + session_id: 'session-123', + config: { langsmith_compat: true } + ) + + expect(attrs['langsmith.span.kind']).to eq 'LLM' + end + + it 'includes temperature when provided' do + attrs = RubyLLM::Instrumentation::SpanBuilder.build_request_attributes( + model: model, + provider: :openai, + session_id: 'session-123', + config: { temperature: 0.7 } + ) + + expect(attrs['gen_ai.request.temperature']).to eq 0.7 + end + end + + describe '.build_metadata_attributes' do + it 'builds attributes with the given prefix' do + attrs = {} + RubyLLM::Instrumentation::SpanBuilder.build_metadata_attributes( + attrs, + { user_id: 123, request_id: 'abc', active: true, score: 0.95 }, + prefix: 'langsmith.metadata' + ) + + expect(attrs['langsmith.metadata.user_id']).to eq 123 + expect(attrs['langsmith.metadata.request_id']).to eq 'abc' + expect(attrs['langsmith.metadata.active']).to be true + expect(attrs['langsmith.metadata.score']).to eq 0.95 + end + + it 'supports custom prefixes' do + attrs = {} + RubyLLM::Instrumentation::SpanBuilder.build_metadata_attributes( + attrs, + { user_id: 123 }, + prefix: 'app.metadata' + ) + + expect(attrs['app.metadata.user_id']).to eq 123 + end + + it 'skips nil values' do + attrs = {} + RubyLLM::Instrumentation::SpanBuilder.build_metadata_attributes( + attrs, + { user_id: 123, empty: nil }, + prefix: 'test' + ) + + expect(attrs).to have_key('test.user_id') + expect(attrs).not_to have_key('test.empty') + end + + it 'passes values through for OTel SDK to handle' do + attrs = {} + hash_value = { nested: 'value' } + array_value = %w[foo bar] + + RubyLLM::Instrumentation::SpanBuilder.build_metadata_attributes( + attrs, + { config: hash_value, tags: array_value }, + prefix: 'test' + ) + + # Values passed through as-is; OTel SDK handles type coercion + expect(attrs['test.config']).to eq hash_value + expect(attrs['test.tags']).to eq array_value + end + end + end + + describe 'Sampling-aware behavior' do + include_context 'with configured RubyLLM' + + let(:non_recording_span) do + instance_double( + OpenTelemetry::Trace::Span, + recording?: false, + add_attributes: nil, + set_attribute: nil, + record_exception: nil, + 'status=': nil + ) + end + + let(:recording_span) do + instance_double( + OpenTelemetry::Trace::Span, + recording?: true, + add_attributes: nil, + set_attribute: nil, + record_exception: nil, + 'status=': nil + ) + end + + let(:mock_tracer) do + tracer = instance_double(OpenTelemetry::Trace::Tracer) + allow(tracer).to receive(:in_span).and_yield(non_recording_span).and_return(nil) + tracer + end + + before do + RubyLLM.configure do |config| + config.tracing_enabled = true + config.tracing_log_content = true + end + allow(described_class).to receive(:tracer).and_return(mock_tracer) + end + + after do + RubyLLM.configure do |config| + config.tracing_enabled = false + config.tracing_log_content = false + end + described_class.reset! + end + + it 'skips attribute building when span is not recording (sampled out)' do + chat = RubyLLM::Chat.new(model: 'gpt-4o-mini', assume_model_exists: true, provider: :openai) + + # Mock provider to return a response + mock_response = RubyLLM::Message.new( + role: :assistant, + content: 'Hello!', + input_tokens: 10, + output_tokens: 5, + model_id: 'gpt-4o-mini' + ) + allow(chat.instance_variable_get(:@provider)).to receive(:complete).and_return(mock_response) + + chat.ask('Hello') + + # When recording? is false, add_attributes should NOT be called + expect(non_recording_span).not_to have_received(:add_attributes) + end + + it 'adds attributes when span is recording' do + allow(mock_tracer).to receive(:in_span).and_yield(recording_span).and_return(nil) + + chat = RubyLLM::Chat.new(model: 'gpt-4o-mini', assume_model_exists: true, provider: :openai) + + mock_response = RubyLLM::Message.new( + role: :assistant, + content: 'Hello!', + input_tokens: 10, + output_tokens: 5, + model_id: 'gpt-4o-mini' + ) + allow(chat.instance_variable_get(:@provider)).to receive(:complete).and_return(mock_response) + + chat.ask('Hello') + + # When recording? is true, add_attributes should be called + expect(recording_span).to have_received(:add_attributes).at_least(:once) + end + end + + describe 'OpenTelemetry Integration', :vcr do + include_context 'with configured RubyLLM' + + let(:exporter) { OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new } + + # Use anonymous class to avoid leaking constants + let(:weather_tool) do + Class.new(RubyLLM::Tool) do + description 'Gets current weather for a location' + param :latitude, desc: 'Latitude (e.g., 52.5200)' + param :longitude, desc: 'Longitude (e.g., 13.4050)' + + def execute(latitude:, longitude:) + "Current weather at #{latitude}, #{longitude}: 15°C, Wind: 10 km/h" + end + + def self.name + 'Weather' + end + end + end + + before do + # Reset any existing OTel configuration + OpenTelemetry::SDK.configure do |c| + c.add_span_processor( + OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(exporter) + ) + end + + # Only configure tracing options - don't overwrite API keys + RubyLLM.configure do |config| + config.tracing_enabled = true + config.tracing_log_content = true + end + + # Reset the cached tracer + described_class.reset! + end + + after do + RubyLLM.configure do |config| + config.tracing_enabled = false + config.tracing_log_content = false + end + described_class.reset! + exporter.reset + end + + it 'creates spans with correct attributes when tracing is enabled' do + VCR.use_cassette('instrumentation_creates_spans_for_chat') do + chat = RubyLLM.chat(model: 'gpt-4.1-nano', assume_model_exists: true, provider: :openai) + chat.ask('Hello') + end + + spans = exporter.finished_spans + expect(spans.count).to be >= 1 + + chat_span = spans.find { |s| s.name == 'ruby_llm.chat' } + expect(chat_span).not_to be_nil + expect(chat_span.kind).to eq(:client) + expect(chat_span.attributes['gen_ai.provider.name']).to eq('openai') + expect(chat_span.attributes['gen_ai.operation.name']).to eq('chat') + expect(chat_span.attributes['gen_ai.conversation.id']).to be_a(String) + + # Check gen_ai.input.messages is valid JSON with correct structure + input_messages = JSON.parse(chat_span.attributes['gen_ai.input.messages']) + expect(input_messages).to be_an(Array) + expect(input_messages.last['role']).to eq('user') + expect(input_messages.last['parts'][0]['content']).to eq('Hello') + end + + it 'creates tool spans as children when tools are used' do + VCR.use_cassette('instrumentation_creates_tool_spans') do + chat = RubyLLM.chat(model: 'gpt-4.1-nano', assume_model_exists: true, provider: :openai) + chat.with_tool(weather_tool) + chat.ask("What's the weather in Berlin? (52.5200, 13.4050)") + end + + spans = exporter.finished_spans + tool_span = spans.find { |s| s.name == 'ruby_llm.tool' } + + expect(tool_span).not_to be_nil + expect(tool_span.kind).to eq(:internal) + expect(tool_span.attributes['gen_ai.tool.name']).to be_a(String) + end + + it 'includes token usage in spans' do + VCR.use_cassette('instrumentation_includes_token_usage') do + chat = RubyLLM.chat(model: 'gpt-4.1-nano', assume_model_exists: true, provider: :openai) + chat.ask('Hello') + end + + spans = exporter.finished_spans + chat_span = spans.find { |s| s.name == 'ruby_llm.chat' } + + expect(chat_span.attributes['gen_ai.usage.input_tokens']).to be > 0 + expect(chat_span.attributes['gen_ai.usage.output_tokens']).to be > 0 + end + + it 'maintains session_id across multiple asks' do + VCR.use_cassette('instrumentation_maintains_session_id') do + chat = RubyLLM.chat(model: 'gpt-4.1-nano', assume_model_exists: true, provider: :openai) + chat.ask("What's your favorite color?") + chat.ask('Why is that?') + end + + spans = exporter.finished_spans + chat_spans = spans.select { |s| s.name == 'ruby_llm.chat' } + + expect(chat_spans.count).to eq(2) + session_ids = chat_spans.map { |s| s.attributes['gen_ai.conversation.id'] }.uniq + expect(session_ids.count).to eq(1) # Same session_id for both + end + + it 'records errors on spans when API calls fail' do + chat = RubyLLM.chat(model: 'gpt-4.1-nano', assume_model_exists: true, provider: :openai) + + # Stub the provider to raise an error + allow(chat.instance_variable_get(:@provider)).to receive(:complete).and_raise( + RubyLLM::ServerError.new(nil, 'Test API error') + ) + + expect { chat.ask('Hello') }.to raise_error(RubyLLM::ServerError) + + spans = exporter.finished_spans + chat_span = spans.find { |s| s.name == 'ruby_llm.chat' } + + expect(chat_span).not_to be_nil + expect(chat_span.status.code).to eq(OpenTelemetry::Trace::Status::ERROR) + expect(chat_span.events.any? { |e| e.name == 'exception' }).to be true + end + + describe 'Streaming' do + it 'creates spans for streaming responses' do + chat = RubyLLM.chat(model: 'gpt-4.1-nano', assume_model_exists: true, provider: :openai) + + # Mock streaming response - provider.complete returns a Message even for streaming + mock_response = RubyLLM::Message.new( + role: :assistant, + content: 'Hello there!', + input_tokens: 8, + output_tokens: 3, + model_id: 'gpt-4.1-nano' + ) + allow(chat.instance_variable_get(:@provider)).to receive(:complete).and_return(mock_response) + + chunks = [] + chat.ask('Hello') { |chunk| chunks << chunk } + + spans = exporter.finished_spans + chat_span = spans.find { |s| s.name == 'ruby_llm.chat' } + + expect(chat_span).not_to be_nil + expect(chat_span.kind).to eq(:client) + expect(chat_span.attributes['gen_ai.provider.name']).to eq('openai') + expect(chat_span.attributes['gen_ai.usage.input_tokens']).to eq(8) + expect(chat_span.attributes['gen_ai.usage.output_tokens']).to eq(3) + end + + it 'includes completion content in streaming spans when content logging enabled' do + chat = RubyLLM.chat(model: 'gpt-4.1-nano', assume_model_exists: true, provider: :openai) + + mock_response = RubyLLM::Message.new( + role: :assistant, + content: 'Streamed response content', + input_tokens: 5, + output_tokens: 4, + model_id: 'gpt-4.1-nano' + ) + allow(chat.instance_variable_get(:@provider)).to receive(:complete).and_return(mock_response) + + chat.ask('Test') { |_chunk| nil } + + spans = exporter.finished_spans + chat_span = spans.find { |s| s.name == 'ruby_llm.chat' } + + output_messages = JSON.parse(chat_span.attributes['gen_ai.output.messages']) + expect(output_messages[0]['parts'][0]['content']).to eq('Streamed response content') + end + + it 'maintains session_id for streaming calls' do + chat = RubyLLM.chat(model: 'gpt-4.1-nano', assume_model_exists: true, provider: :openai) + session_id = chat.session_id + + mock_response = RubyLLM::Message.new( + role: :assistant, + content: 'Response', + model_id: 'gpt-4.1-nano' + ) + allow(chat.instance_variable_get(:@provider)).to receive(:complete).and_return(mock_response) + + chat.ask('First') { |_| nil } + chat.ask('Second') { |_| nil } + + spans = exporter.finished_spans + chat_spans = spans.select { |s| s.name == 'ruby_llm.chat' } + + expect(chat_spans.count).to eq(2) + expect(chat_spans.all? { |s| s.attributes['gen_ai.conversation.id'] == session_id }).to be true + end + + it 'creates tool spans as children during streaming with tools' do + chat = RubyLLM.chat(model: 'gpt-4.1-nano', assume_model_exists: true, provider: :openai) + chat.with_tool(weather_tool) + + # First response triggers tool call + tool_call_response = RubyLLM::Message.new( + role: :assistant, + content: nil, + tool_calls: { + 'call_123' => RubyLLM::ToolCall.new( + id: 'call_123', + name: 'weather', + arguments: { latitude: '52.52', longitude: '13.41' } + ) + }, + model_id: 'gpt-4.1-nano' + ) + + # Second response is final answer + final_response = RubyLLM::Message.new( + role: :assistant, + content: 'The weather in Berlin is nice!', + model_id: 'gpt-4.1-nano' + ) + + call_count = 0 + allow(chat.instance_variable_get(:@provider)).to receive(:complete) do + call_count += 1 + call_count == 1 ? tool_call_response : final_response + end + + chat.ask('Weather in Berlin?') { |_| nil } + + spans = exporter.finished_spans + tool_span = spans.find { |s| s.name == 'ruby_llm.tool' } + chat_spans = spans.select { |s| s.name == 'ruby_llm.chat' } + + expect(chat_spans.count).to eq(2) # Initial call + follow-up after tool + expect(tool_span).not_to be_nil + expect(tool_span.attributes['gen_ai.tool.name']).to eq('weather') + + # Tool span should be child of first chat span + first_chat_span = chat_spans.min_by(&:start_timestamp) + expect(tool_span.parent_span_id).to eq(first_chat_span.span_id) + end + end + end +end