FIX: Use underlying_model_name for evaluation identifier and add param_fallbacks

pyrit/identifiers/evaluation_identifier.py

@@ -40,18 +40,25 @@ class ChildEvalRule:
     * ``included_item_values`` — for list-valued children, only include items
       whose ``params`` match **all** specified key-value pairs. ``None``
       means include all items.
+    * ``param_fallbacks`` — maps a primary param key to a fallback key.
+      When the primary key's value is falsy (empty string, ``None``, or
+      missing), the fallback key's value from the component's raw params
+      is used instead. This keeps fallback logic in the eval layer without
+      changing full component hashes.  ``None`` means no fallbacks.
     """
 
     exclude: bool = False
     included_params: Optional[frozenset[str]] = None
     included_item_values: Optional[dict[str, Any]] = field(default=None)
+    param_fallbacks: Optional[dict[str, str]] = field(default=None)
 
 
 def _build_eval_dict(
     identifier: ComponentIdentifier,
     *,
     child_eval_rules: dict[str, ChildEvalRule],
     _included_params: Optional[frozenset[str]] = None,
+    _param_fallbacks: Optional[dict[str, str]] = None,
 ) -> dict[str, Any]:
     """
     Build a filtered dictionary for eval-hash computation.
@@ -67,6 +74,10 @@ def _build_eval_dict(
         _included_params (Optional[frozenset[str]]): Internal. If set, only
             include params whose keys are in this frozenset. Passed down from
             a parent rule's ``included_params``.
+        _param_fallbacks (Optional[dict[str, str]]): Internal. Maps a primary
+            param key to a fallback key. When the primary value is falsy,
+            the fallback key's value from raw params is used instead.
+            Passed down from a parent rule's ``param_fallbacks``.
 
     Returns:
         dict[str, Any]: The filtered dictionary suitable for hashing.
@@ -84,6 +95,16 @@ def _build_eval_dict(
         }
     )
 
+    # Apply fallbacks: when a primary param is missing or empty string,
+    # substitute with the fallback key's value from the raw params.
+    if _param_fallbacks:
+        for primary_key, fallback_key in _param_fallbacks.items():
+            primary_value = eval_dict.get(primary_key)
+            if primary_value is None or primary_value == "":
+                fallback_value = identifier.params.get(fallback_key)
+                if fallback_value is not None and fallback_value != "":
+                    eval_dict[primary_key] = fallback_value
+
     if identifier.children:
         eval_children: dict[str, Any] = {}
         for name in sorted(identifier.children):
@@ -99,14 +120,17 @@ def _build_eval_dict(
                 required = rule.included_item_values
                 child_list = [c for c in child_list if all(c.params.get(k) == v for k, v in required.items())]
 
-            # For children with a rule, apply included_params; otherwise None → all params kept.
+            # For children with a rule, apply included_params and param_fallbacks;
+            # otherwise None → all params kept, no fallbacks.
             child_included_params = rule.included_params if rule else None
+            child_param_fallbacks = rule.param_fallbacks if rule else None
             hashes = [
                 config_hash(
                     _build_eval_dict(
                         c,
                         child_eval_rules=child_eval_rules,
                         _included_params=child_included_params,
+                        _param_fallbacks=child_param_fallbacks,
                     )
                 )
                 for c in child_list
@@ -208,13 +232,14 @@ class ScorerEvaluationIdentifier(EvaluationIdentifier):
     Evaluation identity for scorers.
 
     The ``prompt_target`` child is filtered to behavioral params only
-    (``model_name``, ``temperature``, ``top_p``), so the same scorer
+    (``underlying_model_name``, ``temperature``, ``top_p``), so the same scorer
     configuration on different deployments produces the same eval hash.
     """
 
     CHILD_EVAL_RULES: ClassVar[dict[str, ChildEvalRule]] = {
         "prompt_target": ChildEvalRule(
-            included_params=frozenset({"model_name", "temperature", "top_p"}),
+            included_params=frozenset({"underlying_model_name", "temperature", "top_p"}),
+            param_fallbacks={"underlying_model_name": "model_name"},
         ),
     }
 
@@ -231,7 +256,7 @@ class AtomicAttackEvaluationIdentifier(EvaluationIdentifier):
       ``objective_scorer``, ``technique_seeds``) are processed recursively
       using the same rules dict, so the rules below apply at any depth.
     * ``objective_target`` — include only ``temperature``.
-    * ``adversarial_chat`` — include ``model_name``, ``temperature``, ``top_p``.
+    * ``adversarial_chat`` — include ``underlying_model_name``, ``temperature``, ``top_p``.
     * ``objective_scorer`` — excluded entirely.
 
     Non-target children (e.g., ``request_converters``, ``response_converters``,
@@ -243,7 +268,8 @@ class AtomicAttackEvaluationIdentifier(EvaluationIdentifier):
             included_params=frozenset({"temperature"}),
         ),
         "adversarial_chat": ChildEvalRule(
-            included_params=frozenset({"model_name", "temperature", "top_p"}),
+            included_params=frozenset({"underlying_model_name", "temperature", "top_p"}),
+            param_fallbacks={"underlying_model_name": "model_name"},
         ),
         "objective_scorer": ChildEvalRule(exclude=True),
         "seed_identifiers": ChildEvalRule(exclude=True),

tests/unit/identifiers/test_atomic_attack_identifier.py

@@ -207,7 +207,8 @@ def test_objective_target_rule(self):
 
     def test_adversarial_chat_rule(self):
         rule = AtomicAttackEvaluationIdentifier.CHILD_EVAL_RULES["adversarial_chat"]
-        assert rule.included_params == frozenset({"model_name", "temperature", "top_p"})
+        assert rule.included_params == frozenset({"underlying_model_name", "temperature", "top_p"})
+        assert rule.param_fallbacks == {"underlying_model_name": "model_name"}
         assert not rule.exclude
 
     def test_scorer_only_keys_absent(self):

tests/unit/identifiers/test_evaluation_identifier.py

@@ -321,6 +321,124 @@ def test_eval_hash_preserved_through_double_roundtrip(self):
         assert _StubEvaluationIdentifier(r2).eval_hash == correct_eval_hash
 
 
+class TestParamFallbacks:
+    """Tests for ChildEvalRule.param_fallbacks in _build_eval_dict."""
+
+    _RULES_WITH_FALLBACK: dict[str, ChildEvalRule] = {
+        "prompt_target": ChildEvalRule(
+            included_params=frozenset({"underlying_model_name", "temperature"}),
+            param_fallbacks={"underlying_model_name": "model_name"},
+        ),
+    }
+
+    def test_primary_param_used_when_present(self):
+        """Test that the primary param value is used when it is non-empty."""
+        child = ComponentIdentifier(
+            class_name="Target",
+            class_module="pyrit.target",
+            params={"underlying_model_name": "gpt-4o", "model_name": "deploy-1", "temperature": 0.7},
+        )
+        identifier = ComponentIdentifier(
+            class_name="Scorer",
+            class_module="pyrit.score",
+            children={"prompt_target": child},
+        )
+
+        result = _build_eval_dict(identifier, child_eval_rules=self._RULES_WITH_FALLBACK)
+        # The child hash should be based on underlying_model_name="gpt-4o", not model_name
+        assert "children" in result
+
+    def test_fallback_used_when_primary_empty(self):
+        """Test that fallback param used when primary is empty string."""
+        child_with_underlying = ComponentIdentifier(
+            class_name="Target",
+            class_module="pyrit.target",
+            params={"underlying_model_name": "gpt-4o", "model_name": "deploy-1", "temperature": 0.7},
+        )
+        child_with_fallback = ComponentIdentifier(
+            class_name="Target",
+            class_module="pyrit.target",
+            params={"underlying_model_name": "", "model_name": "gpt-4o", "temperature": 0.7},
+        )
+        id1 = ComponentIdentifier(
+            class_name="Scorer",
+            class_module="pyrit.score",
+            children={"prompt_target": child_with_underlying},
+        )
+        id2 = ComponentIdentifier(
+            class_name="Scorer",
+            class_module="pyrit.score",
+            children={"prompt_target": child_with_fallback},
+        )
+
+        result1 = _build_eval_dict(id1, child_eval_rules=self._RULES_WITH_FALLBACK)
+        result2 = _build_eval_dict(id2, child_eval_rules=self._RULES_WITH_FALLBACK)
+
+        assert result1["children"]["prompt_target"] == result2["children"]["prompt_target"]
+
+    def test_fallback_used_when_primary_missing(self):
+        """Test that fallback param used when primary key is absent."""
+        child_with_underlying = ComponentIdentifier(
+            class_name="Target",
+            class_module="pyrit.target",
+            params={"underlying_model_name": "gpt-4o", "temperature": 0.7},
+        )
+        child_with_model_name_only = ComponentIdentifier(
+            class_name="Target",
+            class_module="pyrit.target",
+            params={"model_name": "gpt-4o", "temperature": 0.7},
+        )
+        id1 = ComponentIdentifier(
+            class_name="Scorer",
+            class_module="pyrit.score",
+            children={"prompt_target": child_with_underlying},
+        )
+        id2 = ComponentIdentifier(
+            class_name="Scorer",
+            class_module="pyrit.score",
+            children={"prompt_target": child_with_model_name_only},
+        )
+
+        result1 = _build_eval_dict(id1, child_eval_rules=self._RULES_WITH_FALLBACK)
+        result2 = _build_eval_dict(id2, child_eval_rules=self._RULES_WITH_FALLBACK)
+
+        assert result1["children"]["prompt_target"] == result2["children"]["prompt_target"]
+
+    def test_no_fallback_when_no_rules(self):
+        """Test that param_fallbacks=None means no fallback applied."""
+        rules_without_fallback: dict[str, ChildEvalRule] = {
+            "prompt_target": ChildEvalRule(
+                included_params=frozenset({"underlying_model_name", "temperature"}),
+            ),
+        }
+        child_with = ComponentIdentifier(
+            class_name="Target",
+            class_module="pyrit.target",
+            params={"underlying_model_name": "gpt-4o", "temperature": 0.7},
+        )
+        child_without = ComponentIdentifier(
+            class_name="Target",
+            class_module="pyrit.target",
+            params={"model_name": "gpt-4o", "temperature": 0.7},
+        )
+        id1 = ComponentIdentifier(
+            class_name="Scorer",
+            class_module="pyrit.score",
+            children={"prompt_target": child_with},
+        )
+        id2 = ComponentIdentifier(
+            class_name="Scorer",
+            class_module="pyrit.score",
+            children={"prompt_target": child_without},
+        )
+
+        result1 = _build_eval_dict(id1, child_eval_rules=rules_without_fallback)
+        result2 = _build_eval_dict(id2, child_eval_rules=rules_without_fallback)
+
+        # Without fallback, these should produce different hashes
+        assert result1["children"]["prompt_target"] != result2["children"]["prompt_target"]
+
+
 def test_compute_eval_hash_raises_when_hash_none_and_no_rules():
     identifier = ComponentIdentifier.__new__(ComponentIdentifier)
     object.__setattr__(identifier, "hash", None)