Add API to run memory heap dump (#6973)

Author: erosselli

src/fides/api/api/v1/endpoints/admin.py

@@ -8,7 +8,12 @@
 from fides.api.db.database import configure_db, migrate_db, reset_db
 from fides.api.oauth.utils import verify_oauth_client_prod
 from fides.api.util.api_router import APIRouter
+from fides.api.util.memory_watchdog import (
+    _capture_heap_dump,
+    get_memory_watchdog_enabled,
+)
 from fides.common.api import scope_registry
+from fides.common.api.scope_registry import HEAP_DUMP_EXEC
 from fides.config import CONFIG
 
 ADMIN_ROUTER = APIRouter(prefix=API_PREFIX, tags=["Admin"])
@@ -82,3 +87,37 @@ def db_action(action: DBActions, revision: Optional[str] = "head") -> Dict:
             "message": f"Fides database action performed successfully: {action_text}"
         }
     }
+
+
+@ADMIN_ROUTER.post(
+    "/admin/heap-dump",
+    tags=["Admin"],
+    dependencies=[Security(verify_oauth_client_prod, scopes=[HEAP_DUMP_EXEC])],
+    status_code=status.HTTP_200_OK,
+)
+def trigger_heap_dump() -> Dict:
+    """
+    Trigger a heap dump for memory diagnostics.
+
+    Captures and logs detailed memory profiling information including:
+    - Process memory stats (RSS, VMS)
+    - Top object type counts
+    - Garbage collector stats
+    - Uncollectable objects (memory leaks)
+
+    The full heap dump report is logged to error logs.
+    """
+    if not get_memory_watchdog_enabled():
+        raise HTTPException(
+            status_code=status.HTTP_405_METHOD_NOT_ALLOWED,
+            detail="Heap dump functionality is not enabled. Set memory_watchdog_enabled to true in application configuration.",
+        )
+
+    logger.warning("Manual heap dump triggered via API")
+    _capture_heap_dump()
+
+    return {
+        "data": {
+            "message": "Heap dump captured successfully. Check server logs for detailed report."
+        }
+    }

src/fides/api/oauth/roles.py

@@ -16,6 +16,7 @@
     DATA_USE_READ,
     DATASET_READ,
     EVALUATION_READ,
+    HEAP_DUMP_EXEC,
     MASKING_EXEC,
     MASKING_READ,
     MESSAGING_CREATE_OR_UPDATE,
@@ -145,6 +146,7 @@ class RoleRegistryEnum(Enum):
     PRIVACY_REQUEST_NOTIFICATIONS_CREATE_OR_UPDATE,
     PRIVACY_REQUEST_EMAIL_INTEGRATIONS_SEND,
     USER_PERMISSION_ASSIGN_OWNERS,
+    HEAP_DUMP_EXEC,
 ]
 
 ROLES_TO_SCOPES_MAPPING: Dict[str, List] = {

src/fides/api/util/memory_watchdog.py

@@ -80,7 +80,7 @@ def _capture_heap_dump() -> None:
         report_lines = [
             "",  # Leading newline for visual separation
             "=" * 80,
-            "MEMORY DUMP - THRESHOLD EXCEEDED",
+            "MEMORY DUMP",
             "=" * 80,
             "",
         ]

src/fides/common/api/scope_registry.py

@@ -76,6 +76,7 @@
 VIEW_DATA = "view_data"
 WEBHOOK = "webhook"
 WORKER_STATS = "worker-stats"
+HEAP_DUMP = "heap_dump"
 
 ASSIGN_OWNERS = "assign_owners"
 
@@ -146,6 +147,8 @@
 
 ENCRYPTION_EXEC = f"{ENCRYPTION}:{EXEC}"
 
+HEAP_DUMP_EXEC = f"{HEAP_DUMP}:{EXEC}"
+
 EVALUATION_CREATE = f"{EVALUATION}:{CREATE}"
 EVALUATION_READ = f"{EVALUATION}:{READ}"
 EVALUATION_UPDATE = f"{EVALUATION}:{UPDATE}"
@@ -295,6 +298,7 @@
     DATASET_READ: "View datasets",
     DATASET_TEST: "Run a standalone privacy request test for a dataset",
     ENCRYPTION_EXEC: "Encrypt data",
+    HEAP_DUMP_EXEC: "Execute a heap dump for memory diagnostics",
     MESSAGING_TEMPLATE_UPDATE: "Update messaging templates",
     EVALUATION_CREATE: "Create evaluation",
     EVALUATION_READ: "Read evaluations",

tests/ctl/api/test_admin.py

@@ -2,10 +2,36 @@
 import pytest
 from starlette.testclient import TestClient
 
+from fides.api.models.application_config import ApplicationConfig
 from fides.api.util.endpoint_utils import API_PREFIX
 from fides.config import FidesConfig
 
 
+@pytest.fixture(scope="function")
+def memory_watchdog_enabled(db):
+    """Fixture to enable memory watchdog for tests."""
+    ApplicationConfig.create_or_update(
+        db,
+        data={"api_set": {"execution": {"memory_watchdog_enabled": True}}},
+    )
+    yield
+    ApplicationConfig.create_or_update(
+        db,
+        data={"api_set": {"execution": {"memory_watchdog_enabled": False}}},
+    )
+
+
+@pytest.fixture(scope="function")
+def memory_watchdog_disabled(db):
+    """Fixture to explicitly disable memory watchdog for tests."""
+    ApplicationConfig.create_or_update(
+        db,
+        data={"api_set": {"execution": {"memory_watchdog_enabled": False}}},
+    )
+    yield
+    # Reset is handled by default behavior (memory_watchdog_enabled defaults to False)
+
+
 def test_db_reset_dev_mode_enabled(
     test_config: FidesConfig,
     test_client: TestClient,
@@ -36,3 +62,65 @@ def test_db_reset_dev_mode_disabled(
 
     assert response.status_code == 501
     assert response.json()["detail"] == error_message
+
+
+def test_heap_dump_when_disabled(
+    test_config: FidesConfig,
+    test_client: TestClient,
+) -> None:
+    """Test that heap dump returns 405 when memory_watchdog_enabled is False (default)."""
+    response = test_client.post(
+        test_config.cli.server_url + API_PREFIX + "/admin/heap-dump/",
+        headers=test_config.user.auth_header,
+    )
+
+    assert response.status_code == 405
+    assert response.json()["detail"] == (
+        "Heap dump functionality is not enabled. "
+        "Set memory_watchdog_enabled to true in application configuration."
+    )
+
+
+@pytest.mark.usefixtures("memory_watchdog_disabled")
+def test_heap_dump_explicitly_disabled(
+    test_config: FidesConfig,
+    test_client: TestClient,
+) -> None:
+    """Test that heap dump returns 405 when memory_watchdog_enabled is explicitly set to False."""
+    response = test_client.post(
+        test_config.cli.server_url + API_PREFIX + "/admin/heap-dump/",
+        headers=test_config.user.auth_header,
+    )
+
+    assert response.status_code == 405
+    assert response.json()["detail"] == (
+        "Heap dump functionality is not enabled. "
+        "Set memory_watchdog_enabled to true in application configuration."
+    )
+
+
+@pytest.mark.usefixtures("memory_watchdog_enabled")
+def test_heap_dump_logs_heap_stats(
+    test_config: FidesConfig,
+    test_client: TestClient,
+    loguru_caplog,
+) -> None:
+    """Test that heap dump endpoint logs heap statistics."""
+    response = test_client.post(
+        test_config.cli.server_url + API_PREFIX + "/admin/heap-dump/",
+        headers=test_config.user.auth_header,
+    )
+
+    assert response.status_code == 200
+
+    # Verify that the heap dump was logged
+    log_output = loguru_caplog.text
+
+    # Check for the info message that heap dump was triggered
+    assert "Manual heap dump triggered via API" in log_output
+
+    # Check for key sections of the heap dump in logs
+    assert "MEMORY DUMP" in log_output
+    assert "PROCESS MEMORY STATS" in log_output
+    assert "OBJECT TYPE COUNTS (Top 10)" in log_output
+    assert "GARBAGE COLLECTOR STATS" in log_output

tests/ops/util/test_memory_watchdog.py

@@ -481,7 +481,7 @@ def test_capture_heap_dump_single_log_message(
         logged_message = str(mock_logger.error.call_args[0][0])
 
         # Verify all expected sections are in the single message
-        assert "MEMORY DUMP - THRESHOLD EXCEEDED" in logged_message
+        assert "MEMORY DUMP" in logged_message
         assert "PROCESS MEMORY STATS" in logged_message
         assert "RSS (Resident Set Size)" in logged_message
         assert "512.0 MiB" in logged_message