diff --git a/ddtrace/internal/constants.py b/ddtrace/internal/constants.py index 9a5829aa86a..3bc94dd8826 100644 --- a/ddtrace/internal/constants.py +++ b/ddtrace/internal/constants.py @@ -45,8 +45,9 @@ SPAN_API_DATADOG = "datadog" SPAN_API_OTEL = "otel" SPAN_API_OPENTRACING = "opentracing" -DEFAULT_BUFFER_SIZE = 20 << 20 # 20 MB -DEFAULT_MAX_PAYLOAD_SIZE = 20 << 20 # 20 MB +DEFAULT_BUFFER_SIZE = 30 << 20 # 30 MB +DEFAULT_MAX_PAYLOAD_SIZE = 30 << 20 # 30 MB +DEFAULT_FLUSH_MIN_SPANS = 200 DEFAULT_PROCESSING_INTERVAL = 1.0 DEFAULT_REUSE_CONNECTIONS = False BLOCKED_RESPONSE_HTML = """You've been blocked

Sorry, you cannot access this page. Please contact the customer service team.

""" # noqa: E501 diff --git a/ddtrace/settings/_config.py b/ddtrace/settings/_config.py index 56b6857d531..d2ab3e33eb6 100644 --- a/ddtrace/settings/_config.py +++ b/ddtrace/settings/_config.py @@ -26,6 +26,7 @@ from ..internal.constants import _PROPAGATION_STYLE_DEFAULT from ..internal.constants import _PROPAGATION_STYLE_NONE from ..internal.constants import DEFAULT_BUFFER_SIZE +from ..internal.constants import DEFAULT_FLUSH_MIN_SPANS from ..internal.constants import DEFAULT_MAX_PAYLOAD_SIZE from ..internal.constants import DEFAULT_PROCESSING_INTERVAL from ..internal.constants import DEFAULT_REUSE_CONNECTIONS @@ -455,7 +456,7 @@ def __init__(self): self._trace_rate_limit, ) self._partial_flush_enabled = _get_config("DD_TRACE_PARTIAL_FLUSH_ENABLED", True, asbool) - self._partial_flush_min_spans = _get_config("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", 300, int) + self._partial_flush_min_spans = _get_config("DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", DEFAULT_FLUSH_MIN_SPANS, int) self._http = HttpConfig(header_tags=self._trace_http_header_tags) self._remote_config_enabled = _get_config("DD_REMOTE_CONFIGURATION_ENABLED", True, asbool) diff --git a/docs/configuration.rst b/docs/configuration.rst index 79a8a66a0dd..c53a1048c50 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -246,7 +246,7 @@ Traces DD_TRACE_PARTIAL_FLUSH_MIN_SPANS: type: Integer - default: 300 + default: 200 description: Maximum number of spans sent per trace per payload when ``DD_TRACE_PARTIAL_FLUSH_ENABLED=True``. DD_TRACE_PROPAGATION_EXTRACT_FIRST: @@ -307,7 +307,7 @@ Traces DD_TRACE_WRITER_BUFFER_SIZE_BYTES: type: Int - default: 8388608 + default: 31457280 description: The max size in bytes of traces to buffer between flushes to the agent. DD_TRACE_WRITER_INTERVAL_SECONDS: @@ -317,7 +317,7 @@ Traces DD_TRACE_WRITER_MAX_PAYLOAD_SIZE_BYTES: type: Int - default: 8388608 + default: 31457280 description: | The max size in bytes of each payload item sent to the trace agent. If the max payload size is greater than buffer size, diff --git a/releasenotes/notes/update-span-buffer-sizes-18552a87bf54f8ef.yaml b/releasenotes/notes/update-span-buffer-sizes-18552a87bf54f8ef.yaml new file mode 100644 index 00000000000..7f872610a96 --- /dev/null +++ b/releasenotes/notes/update-span-buffer-sizes-18552a87bf54f8ef.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + - Increase ``DD_TRACE_BUFFER_SIZE_BYTES`` default value to 30MB to avoid potential OOMs under heavy load situations. + - Decrease ``DD_TRACE_PARTIAL_FLUSH_MIN_SPANS`` to 200. diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index b438d5e3aa8..66f00594d90 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -199,7 +199,7 @@ def test_metrics(): from tests.utils import AnyInt from tests.utils import override_global_config - assert t._span_aggregator.partial_flush_min_spans == 300 + assert t._span_aggregator.partial_flush_min_spans == 30 << 20 with override_global_config(dict(_health_metrics_enabled=True)): statsd_mock = mock.Mock() @@ -652,8 +652,8 @@ def test_writer_configured_correctly_from_env(): def test_writer_configured_correctly_from_env_defaults(): import ddtrace - assert ddtrace.tracer._span_aggregator.writer._encoder.max_size == 20 << 20 - assert ddtrace.tracer._span_aggregator.writer._encoder.max_item_size == 20 << 20 + assert ddtrace.tracer._span_aggregator.writer._encoder.max_size == 30 << 20 + assert ddtrace.tracer._span_aggregator.writer._encoder.max_item_size == 30 << 20 assert ddtrace.tracer._span_aggregator.writer._interval == 1.0 @@ -681,8 +681,8 @@ def test_writer_configured_correctly_from_env_defaults_under_ddtrace_run(ddtrace """ import ddtrace -assert ddtrace.tracer._span_aggregator.writer._encoder.max_size == 20 << 20 -assert ddtrace.tracer._span_aggregator.writer._encoder.max_item_size == 20 << 20 +assert ddtrace.tracer._span_aggregator.writer._encoder.max_size == 30 << 20 +assert ddtrace.tracer._span_aggregator.writer._encoder.max_item_size == 30 << 20 assert ddtrace.tracer._span_aggregator.writer._interval == 1.0 """, )