diff --git a/ddtrace/bootstrap/preload.py b/ddtrace/bootstrap/preload.py index 1dd02d4efe8..f30caeb4a4a 100644 --- a/ddtrace/bootstrap/preload.py +++ b/ddtrace/bootstrap/preload.py @@ -85,12 +85,6 @@ def _otel_signals(_): set_otel_meter_provider() -if config._llmobs_enabled: - from ddtrace.llmobs import LLMObs - - LLMObs.enable(_auto=True) - - @ModuleWatchdog.after_module_imported("gevent.monkey") def _(_): # uWSGI + gevent support: when the application module is imported, the diff --git a/ddtrace/internal/remoteconfig/products/apm_tracing.py b/ddtrace/internal/remoteconfig/products/apm_tracing.py index 9f71c6d87ab..fbed34dc098 100644 --- a/ddtrace/internal/remoteconfig/products/apm_tracing.py +++ b/ddtrace/internal/remoteconfig/products/apm_tracing.py @@ -82,16 +82,19 @@ def __call__(self, payloads: t.Sequence[Payload]) -> None: log.debug("Received APM tracing config payload: %s", payload) config_id = payload.metadata.id - seen_config_ids.add(config_id) if (content := payload.content) is None: log.debug( - "ignoring invalid APM Tracing remote config payload with no content, product: %s, path: %s", + "Removing APM tracing config %s (deleted by agent), product: %s, path: %s", + config_id, payload.metadata.product_name, payload.path, ) + self._config_map.pop(config_id, None) continue + seen_config_ids.add(config_id) + service_target = t.cast(t.Optional[dict], content.get("service_target")) service = t.cast(str, service_target.get("service")) if service_target is not None else None diff --git a/ddtrace/internal/settings/_config.py b/ddtrace/internal/settings/_config.py index b9bf43f7512..bd458d9f259 100644 --- a/ddtrace/internal/settings/_config.py +++ b/ddtrace/internal/settings/_config.py @@ -389,6 +389,16 @@ def _default_config() -> dict[str, _ConfigItem]: envs=["DD_APPSEC_SCA_ENABLED"], modifier=asbool, ), + "_llmobs_enabled": _ConfigItem( + default=False, + envs=["DD_LLMOBS_ENABLED"], + modifier=asbool, + ), + "_llmobs_ml_app": _ConfigItem( + default=None, + envs=["DD_LLMOBS_ML_APP"], + modifier=lambda x: x, + ), } @@ -661,9 +671,7 @@ def __init__(self) -> None: self._dd_app_key = _get_config("DD_APP_KEY", report_telemetry=False) self._dd_site = _get_config("DD_SITE", "datadoghq.com") - self._llmobs_enabled = _get_config("DD_LLMOBS_ENABLED", False, asbool) self._llmobs_sample_rate = _get_config("DD_LLMOBS_SAMPLE_RATE", 1.0, float) - self._llmobs_ml_app = _get_config("DD_LLMOBS_ML_APP") self._llmobs_agentless_enabled = _get_config("DD_LLMOBS_AGENTLESS_ENABLED", None, asbool) self._llmobs_instrumented_proxy_urls = _get_config( "DD_LLMOBS_INSTRUMENTED_PROXY_URLS", None, lambda x: set(x.strip().split(",")) diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index e45206cd6f2..b91da2ec68b 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -158,7 +158,7 @@ log = get_logger(__name__) -SUPPORTED_LLMOBS_INTEGRATIONS = { +SUPPORTED_LLMOBS_INTEGRATIONS: dict[str, str] = { "anthropic": "anthropic", "bedrock": "botocore", "openai": "openai", @@ -173,7 +173,10 @@ "mcp": "mcp", "pydantic_ai": "pydantic_ai", "claude_agent_sdk": "claude_agent_sdk", - # requests/concurrent frameworks for distributed injection/extraction +} + +# requests/concurrent frameworks for distributed injection/extraction +_INTEGRATIONS_W_PROPAGATION_SUPPORT: dict[str, str] = { "requests": "requests", "httpx": "httpx", "urllib3": "urllib3", @@ -1539,9 +1542,12 @@ def register_processor(cls, processor: Optional[Callable[[LLMObsSpan], Optional[ @classmethod def _integration_is_enabled(cls, integration: str) -> bool: - if integration not in SUPPORTED_LLMOBS_INTEGRATIONS: + module_name = SUPPORTED_LLMOBS_INTEGRATIONS.get( + integration, _INTEGRATIONS_W_PROPAGATION_SUPPORT.get(integration) + ) + if module_name is None: return False - return SUPPORTED_LLMOBS_INTEGRATIONS[integration] in ddtrace._monkey._get_patched_modules() + return module_name in ddtrace._monkey._get_patched_modules() @classmethod def disable(cls) -> None: @@ -1797,9 +1803,12 @@ def _patch_integrations() -> None: """ Patch LLM integrations. Ensure that we do not ignore DD_TRACE__ENABLED or DD_PATCH_MODULES settings. """ + llm_integrations: list[str] = list(SUPPORTED_LLMOBS_INTEGRATIONS.values()) + list( + _INTEGRATIONS_W_PROPAGATION_SUPPORT.values() + ) integrations_to_patch: dict[str, Union[list[str], bool]] = { integration: ["bedrock-runtime", "bedrock-agent-runtime"] if integration == "botocore" else True - for integration in SUPPORTED_LLMOBS_INTEGRATIONS.values() + for integration in llm_integrations } for module, _ in integrations_to_patch.items(): env_var = "DD_TRACE_%s_ENABLED" % module.upper() @@ -1808,7 +1817,7 @@ def _patch_integrations() -> None: dd_patch_modules = _env.get("DD_PATCH_MODULES") dd_patch_modules_to_str = parse_tags_str(dd_patch_modules) integrations_to_patch.update( - {k: asbool(v) for k, v in dd_patch_modules_to_str.items() if k in SUPPORTED_LLMOBS_INTEGRATIONS.values()} + {k: asbool(v) for k, v in dd_patch_modules_to_str.items() if k in llm_integrations} ) patch(raise_errors=True, **integrations_to_patch) llm_patched_modules = [k for k, v in integrations_to_patch.items() if v] diff --git a/ddtrace/llmobs/_product.py b/ddtrace/llmobs/_product.py new file mode 100644 index 00000000000..758b18eb947 --- /dev/null +++ b/ddtrace/llmobs/_product.py @@ -0,0 +1,88 @@ +"""LLMObs product — Remote Config support for `llmobs_enabled` / `llmobs_ml_app`. + +Wires into the APM_TRACING RC product via the "apm-tracing.rc" event hub so +that operators can enable/disable LLMObs and set the ml_app name at runtime +without restarting the service. +""" + +import enum +import typing as t + +from ddtrace.internal.logger import get_logger + + +log = get_logger(__name__) + +requires = ["apm-tracing-rc"] + + +class APMCapabilities(enum.IntFlag): + APM_TRACING_LLMOBS = 1 << 48 + + +def post_preload() -> None: + pass + + +def start() -> None: + from ddtrace import config + + if config._llmobs_enabled: + from ddtrace.llmobs import LLMObs + + LLMObs.enable(_auto=True) + + record_llm_oneclick_support() + + +def restart(join: bool = False) -> None: + pass + + +def stop(join: bool = False) -> None: + pass + + +def record_llm_oneclick_support() -> None: + """Track LLM integrations detected in the environment.""" + from ddtrace import config + from ddtrace.internal.module import is_module_installed + from ddtrace.internal.telemetry import telemetry_writer + from ddtrace.llmobs._llmobs import SUPPORTED_LLMOBS_INTEGRATIONS + + llm_oneclick_supported: bool = False + if config._remote_config_enabled: + for module_name in SUPPORTED_LLMOBS_INTEGRATIONS.values(): + if is_module_installed(module_name): + llm_oneclick_supported = True + break + + telemetry_writer.add_configuration("llmobs_oneclick_supported", llm_oneclick_supported) + + +def apm_tracing_rc(lib_config: dict, dd_config: t.Any) -> None: + """Handle APM_TRACING RC payloads for LLMObs fields. + + Expected lib_config shape: + llmobs.enabled (bool) — enable or disable LLMObs at runtime. + llmobs.ml_app_name (str) — ML application name; applied alongside enablement. + + Config values are written via the standard remote_config source so they + participate in the normal config precedence chain. + """ + from ddtrace.llmobs import LLMObs + + llmobs_config = lib_config.get("llmobs", {}) + + enabled_config = dd_config._config["_llmobs_enabled"] + enabled_config.set_value(llmobs_config.get("enabled"), "remote_config") + + ml_app_config = dd_config._config["_llmobs_ml_app"] + ml_app_config.set_value(llmobs_config.get("ml_app_name"), "remote_config") + + if enabled_config.value() and not LLMObs.enabled: + log.debug("Enabling LLMObs via Remote Config (ml_app=%r)", ml_app_config.value()) + LLMObs.enable(ml_app=ml_app_config.value(), _auto=True) + elif not enabled_config.value() and LLMObs.enabled: + log.debug("Disabling LLMObs via Remote Config: %r", llmobs_config) + LLMObs.disable() diff --git a/pyproject.toml b/pyproject.toml index 2e4968f4385..6a220ffa1d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ datadog = "ddtrace.contrib.internal.mlflow.auth_plugin:DatadogHeaderProvider" [project.entry-points.'ddtrace.products'] "apm-tracing-rc" = "ddtrace.internal.remoteconfig.products.apm_tracing" +"llmobs" = "ddtrace.llmobs._product" "code-origin-for-spans" = "ddtrace.debugging._products.code_origin.span" "dynamic-instrumentation" = "ddtrace.debugging._products.dynamic_instrumentation" "exception-replay" = "ddtrace.debugging._products.exception_replay" diff --git a/releasenotes/notes/feat-rc-support-llmo-08745314854b5764.yaml b/releasenotes/notes/feat-rc-support-llmo-08745314854b5764.yaml new file mode 100644 index 00000000000..0e1c07cd4fe --- /dev/null +++ b/releasenotes/notes/feat-rc-support-llmo-08745314854b5764.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + llmobs: Adds support for enabling and disabling LLMObs via Remote Configuration. diff --git a/tests/llmobs/test_llmobs_rc.py b/tests/llmobs/test_llmobs_rc.py new file mode 100644 index 00000000000..87d2b2facc4 --- /dev/null +++ b/tests/llmobs/test_llmobs_rc.py @@ -0,0 +1,59 @@ +import pytest + + +@pytest.mark.subprocess(env={"DD_REMOTE_CONFIGURATION_ENABLED": "false"}) +def test_rc_enables_llmobs_and_sets_ml_app(): + """RC payload with llmobs.enabled=true sets config and calls LLMObs.enable().""" + import mock + + import ddtrace + from ddtrace.llmobs import LLMObs + from ddtrace.llmobs.product import apm_tracing_rc + + assert ddtrace.config._llmobs_enabled is False + with mock.patch.object(LLMObs, "enable") as mock_enable: + apm_tracing_rc( + {"llmobs": {"enabled": True, "ml_app_name": "my-llm-app"}}, + ddtrace.config, + ) + + assert ddtrace.config._llmobs_enabled is True + assert ddtrace.config._llmobs_ml_app == "my-llm-app" + mock_enable.assert_called_once_with(ml_app="my-llm-app", _auto=True) + + +@pytest.mark.subprocess(ddtrace_run=True, env={"DD_REMOTE_CONFIGURATION_ENABLED": "false", "DD_LLMOBS_ENABLED": "true"}) +def test_rc_disables_llmobs(): + """RC payload with llmobs.enabled=false calls LLMObs.disable() when LLMObs is running.""" + import mock + + import ddtrace + from ddtrace.llmobs import LLMObs + from ddtrace.llmobs.product import apm_tracing_rc + + assert LLMObs.enabled, "ddtrace-run with DD_LLMOBS_ENABLED=true should have enabled LLMObs" + with mock.patch.object(LLMObs, "disable") as mock_disable: + apm_tracing_rc({"llmobs": {"enabled": False}}, ddtrace.config) + + mock_disable.assert_called_once() + assert ddtrace.config._llmobs_enabled is False + + +@pytest.mark.subprocess( + ddtrace_run=True, env={"DD_REMOTE_CONFIGURATION_ENABLED": "false", "DD_LLMOBS_ENABLED": "false"} +) +def test_rc_missing_llmobs_is_noop(): + """Payloads missing llmobs.enabled do not affect LLMObs state.""" + import mock + + import ddtrace + from ddtrace.llmobs import LLMObs + from ddtrace.llmobs.product import apm_tracing_rc + + for payload in ({}, {"llmobs": {}}, {"llmobs": {"enabled": None}}): + with mock.patch.object(LLMObs, "enable") as mock_enable: + apm_tracing_rc(payload, ddtrace.config) + + mock_enable.assert_not_called() + assert ddtrace.config._llmobs_enabled is False + assert not LLMObs.enabled