From 54dbc8fa1f62e16e2132fd01f721ad22af6467fa Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 30 Jun 2025 10:46:19 +0200 Subject: [PATCH 1/7] Update span buffer and min flush count Signed-off-by: Juanjo Alvarez --- ddtrace/internal/constants.py | 4 ++-- ddtrace/settings/_config.py | 2 +- docs/configuration.rst | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ddtrace/internal/constants.py b/ddtrace/internal/constants.py index 9a5829aa86a..42e1511d959 100644 --- a/ddtrace/internal/constants.py +++ b/ddtrace/internal/constants.py @@ -45,8 +45,8 @@ SPAN_API_DATADOG = "datadog" SPAN_API_OTEL = "otel" SPAN_API_OPENTRACING = "opentracing" -DEFAULT_BUFFER_SIZE = 20 << 20 # 20 MB -DEFAULT_MAX_PAYLOAD_SIZE = 20 << 20 # 20 MB +DEFAULT_BUFFER_SIZE = 30 << 20 # 30 MB +DEFAULT_MAX_PAYLOAD_SIZE = 30 << 20 # 30 MB DEFAULT_PROCESSING_INTERVAL = 1.0 DEFAULT_REUSE_CONNECTIONS = False BLOCKED_RESPONSE_HTML = """You've been blocked

Sorry, you cannot access this page. Please contact the customer service team.

""" # noqa: E501 diff --git a/ddtrace/settings/_config.py b/ddtrace/settings/_config.py index 56b6857d531..17c0026a01a 100644 --- a/ddtrace/settings/_config.py +++ b/ddtrace/settings/_config.py @@ -455,7 +455,7 @@ def __init__(self): self._trace_rate_limit, ) self._partial_flush_enabled = _get_config("DD_TRACE_PARTIAL_FLUSH_ENABLED", True, asbool) - self._partial_flush_min_spans = _get_config("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", 300, int) + self._partial_flush_min_spans = _get_config("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", 200, int) self._http = HttpConfig(header_tags=self._trace_http_header_tags) self._remote_config_enabled = _get_config("DD_REMOTE_CONFIGURATION_ENABLED", True, asbool) diff --git a/docs/configuration.rst b/docs/configuration.rst index 79a8a66a0dd..c53a1048c50 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -246,7 +246,7 @@ Traces DD_TRACE_PARTIAL_FLUSH_MIN_SPANS: type: Integer - default: 300 + default: 200 description: Maximum number of spans sent per trace per payload when ``DD_TRACE_PARTIAL_FLUSH_ENABLED=True``. DD_TRACE_PROPAGATION_EXTRACT_FIRST: @@ -307,7 +307,7 @@ Traces DD_TRACE_WRITER_BUFFER_SIZE_BYTES: type: Int - default: 8388608 + default: 31457280 description: The max size in bytes of traces to buffer between flushes to the agent. DD_TRACE_WRITER_INTERVAL_SECONDS: @@ -317,7 +317,7 @@ Traces DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES: type: Int - default: 8388608 + default: 31457280 description: | The max size in bytes of each payload item sent to the trace agent. If the max payload size is greater than buffer size, From 0147be79d0ae61c8569e6fe7f72598d02afbe829 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 30 Jun 2025 10:49:10 +0200 Subject: [PATCH 2/7] changelog Signed-off-by: Juanjo Alvarez --- .../notes/update-span-buffer-sizes-18552a87bf54f8ef.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 releasenotes/notes/update-span-buffer-sizes-18552a87bf54f8ef.yaml diff --git a/releasenotes/notes/update-span-buffer-sizes-18552a87bf54f8ef.yaml b/releasenotes/notes/update-span-buffer-sizes-18552a87bf54f8ef.yaml new file mode 100644 index 00000000000..7f872610a96 --- /dev/null +++ b/releasenotes/notes/update-span-buffer-sizes-18552a87bf54f8ef.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + - Increase ``DD_TRACE_BUFFER_SIZE_BYTES`` default value to 30MB to avoid potential OOMs under heavy load situations. + - Decrease ``DD_TRACE_PARTIAL_FLUSH_MIN_SPANS`` to 200. From d862b245654d39cab987af12b7d9906760377ef1 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 30 Jun 2025 10:51:16 +0200 Subject: [PATCH 3/7] Move the min flush spans to _constants.py Signed-off-by: Juanjo Alvarez --- ddtrace/internal/constants.py | 1 + ddtrace/settings/_config.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ddtrace/internal/constants.py b/ddtrace/internal/constants.py index 42e1511d959..3bc94dd8826 100644 --- a/ddtrace/internal/constants.py +++ b/ddtrace/internal/constants.py @@ -47,6 +47,7 @@ SPAN_API_OPENTRACING = "opentracing" DEFAULT_BUFFER_SIZE = 30 << 20 # 30 MB DEFAULT_MAX_PAYLOAD_SIZE = 30 << 20 # 30 MB +DEFAULT_FLUSH_MIN_SPANS = 200 DEFAULT_PROCESSING_INTERVAL = 1.0 DEFAULT_REUSE_CONNECTIONS = False BLOCKED_RESPONSE_HTML = """You've been blocked

Sorry, you cannot access this page. Please contact the customer service team.

""" # noqa: E501 diff --git a/ddtrace/settings/_config.py b/ddtrace/settings/_config.py index 17c0026a01a..d2ab3e33eb6 100644 --- a/ddtrace/settings/_config.py +++ b/ddtrace/settings/_config.py @@ -26,6 +26,7 @@ from ..internal.constants import _PROPAGATION_STYLE_DEFAULT from ..internal.constants import _PROPAGATION_STYLE_NONE from ..internal.constants import DEFAULT_BUFFER_SIZE +from ..internal.constants import DEFAULT_FLUSH_MIN_SPANS from ..internal.constants import DEFAULT_MAX_PAYLOAD_SIZE from ..internal.constants import DEFAULT_PROCESSING_INTERVAL from ..internal.constants import DEFAULT_REUSE_CONNECTIONS @@ -455,7 +456,7 @@ def __init__(self): self._trace_rate_limit, ) self._partial_flush_enabled = _get_config("DD_TRACE_PARTIAL_FLUSH_ENABLED", True, asbool) - self._partial_flush_min_spans = _get_config("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", 200, int) + self._partial_flush_min_spans = _get_config("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", DEFAULT_FLUSH_MIN_SPANS, int) self._http = HttpConfig(header_tags=self._trace_http_header_tags) self._remote_config_enabled = _get_config("DD_REMOTE_CONFIGURATION_ENABLED", True, asbool) From c47e04e354bd5426d6f9e5146099d1ee96e9dd0e Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 30 Jun 2025 12:12:04 +0200 Subject: [PATCH 4/7] Update values also in tests Signed-off-by: Juanjo Alvarez --- tests/integration/test_integration.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index b438d5e3aa8..f577f399b5e 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -8,6 +8,7 @@ import pytest from ddtrace.internal.atexit import register_on_exit_signal +from ddtrace.internal.constants import DEFAULT_FLUSH_MIN_SPANS from tests.integration.utils import import_ddtrace_in_subprocess from tests.integration.utils import parametrize_with_all_encodings from tests.integration.utils import skip_if_testagent @@ -199,7 +200,7 @@ def test_metrics(): from tests.utils import AnyInt from tests.utils import override_global_config - assert t._span_aggregator.partial_flush_min_spans == 300 + assert t._span_aggregator.partial_flush_min_spans == DEFAULT_FLUSH_MIN_SPANS with override_global_config(dict(_health_metrics_enabled=True)): statsd_mock = mock.Mock() @@ -652,8 +653,8 @@ def test_writer_configured_correctly_from_env(): def test_writer_configured_correctly_from_env_defaults(): import ddtrace - assert ddtrace.tracer._span_aggregator.writer._encoder.max_size == 20 << 20 - assert ddtrace.tracer._span_aggregator.writer._encoder.max_item_size == 20 << 20 + assert ddtrace.tracer._span_aggregator.writer._encoder.max_size == 30 << 20 + assert ddtrace.tracer._span_aggregator.writer._encoder.max_item_size == 30 << 20 assert ddtrace.tracer._span_aggregator.writer._interval == 1.0 From ea5b03d25627244b7066c690478440d89f2a0c97 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 30 Jun 2025 14:21:41 +0200 Subject: [PATCH 5/7] Update values also in tests Signed-off-by: Juanjo Alvarez --- tests/integration/test_integration.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index f577f399b5e..f6b97b1e056 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -8,7 +8,6 @@ import pytest from ddtrace.internal.atexit import register_on_exit_signal -from ddtrace.internal.constants import DEFAULT_FLUSH_MIN_SPANS from tests.integration.utils import import_ddtrace_in_subprocess from tests.integration.utils import parametrize_with_all_encodings from tests.integration.utils import skip_if_testagent @@ -200,7 +199,7 @@ def test_metrics(): from tests.utils import AnyInt from tests.utils import override_global_config - assert t._span_aggregator.partial_flush_min_spans == DEFAULT_FLUSH_MIN_SPANS + assert t._span_aggregator.partial_flush_min_spans == 30<<20 with override_global_config(dict(_health_metrics_enabled=True)): statsd_mock = mock.Mock() From 57a0e2102c912d1b5dc5e830be9b2dbc72f8d337 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 30 Jun 2025 15:21:26 +0200 Subject: [PATCH 6/7] fmt Signed-off-by: Juanjo Alvarez --- tests/integration/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index f6b97b1e056..b49805586c5 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -199,7 +199,7 @@ def test_metrics(): from tests.utils import AnyInt from tests.utils import override_global_config - assert t._span_aggregator.partial_flush_min_spans == 30<<20 + assert t._span_aggregator.partial_flush_min_spans == 30 << 20 with override_global_config(dict(_health_metrics_enabled=True)): statsd_mock = mock.Mock() From 205c70ff690c170c0ab5c5fc480e3e9306ad4317 Mon Sep 17 00:00:00 2001 From: Juanjo Alvarez Date: Mon, 30 Jun 2025 16:35:24 +0200 Subject: [PATCH 7/7] Update values also in tests Signed-off-by: Juanjo Alvarez --- tests/integration/test_integration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index b49805586c5..66f00594d90 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -681,8 +681,8 @@ def test_writer_configured_correctly_from_env_defaults_under_ddtrace_run(ddtrace """ import ddtrace -assert ddtrace.tracer._span_aggregator.writer._encoder.max_size == 20 << 20 -assert ddtrace.tracer._span_aggregator.writer._encoder.max_item_size == 20 << 20 +assert ddtrace.tracer._span_aggregator.writer._encoder.max_size == 30 << 20 +assert ddtrace.tracer._span_aggregator.writer._encoder.max_item_size == 30 << 20 assert ddtrace.tracer._span_aggregator.writer._interval == 1.0 """, )