fix: preflight check now validates reasoning_content for thinking models

.github/run-eval/resolve_model_config.py

@@ -330,9 +330,16 @@ def check_model(
             **kwargs,
         )
 
-        content = response.choices[0].message.content if response.choices else None
+        response_content = (
+            response.choices[0].message.content if response.choices else None
+        )
+        reasoning_content = (
+            getattr(response.choices[0].message, "reasoning_content", None)
+            if response.choices
+            else None
+        )
 
-        if content:
+        if response_content or reasoning_content:
             return True, f"✓ {display_name}: OK"
         else:
             # Check if there's any other data in the response for diagnostics

tests/github_workflows/test_resolve_model_config.py

@@ -297,7 +297,9 @@ def test_empty_response(self):
             "llm_config": {"model": "litellm_proxy/test-model"},
         }
         mock_response = MagicMock()
-        mock_response.choices = [MagicMock(message=MagicMock(content=""))]
+        mock_response.choices = [
+            MagicMock(message=MagicMock(content="", reasoning_content=None))
+        ]
 
         with patch("litellm.completion", return_value=mock_response):
             success, message = test_model(model_config, "test-key", "https://test.com")
@@ -306,6 +308,48 @@ def test_empty_response(self):
         assert "✗" in message
         assert "Empty response" in message
 
+    def test_thinking_model_success(self):
+        """Test that a thinking model with only reasoning_content passes."""
+        model_config = {
+            "display_name": "Thinking Model",
+            "llm_config": {"model": "litellm_proxy/thinking-model"},
+        }
+        mock_response = MagicMock()
+        mock_response.choices = [
+            MagicMock(
+                message=MagicMock(content="", reasoning_content="Let me think...")
+            )
+        ]
+
+        with patch("litellm.completion", return_value=mock_response):
+            success, message = test_model(model_config, "test-key", "https://test.com")
+
+        assert success is True
+        assert "✓" in message
+
+    def test_model_without_reasoning_content_attribute(self):
+        """Test that models whose Message object lacks reasoning_content don't raise."""
+        from types import SimpleNamespace
+
+        model_config = {
+            "display_name": "Standard Model",
+            "llm_config": {"model": "litellm_proxy/standard-model"},
+        }
+        mock_response = MagicMock()
+        # SimpleNamespace has only the attributes we give it - no reasoning_content
+        message = SimpleNamespace(content="2")
+        choice = MagicMock()
+        choice.message = message
+        mock_response.choices = [choice]
+
+        with patch("litellm.completion", return_value=mock_response):
+            success, message_str = test_model(
+                model_config, "test-key", "https://test.com"
+            )
+
+        assert success is True
+        assert "✓" in message_str
+
     def test_timeout_error(self):
         """Test that timeout errors are handled correctly."""
         import litellm