Updating num_runs

xzrderek · xzrderek · commit a473f0c0b7e4 · 2025-08-19T10:28:33.000-07:00
diff --git a/eval_protocol/benchmarks/test_gpqa.py b/eval_protocol/benchmarks/test_gpqa.py
@@ -97,7 +97,7 @@ def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) ->
     rollout_processor=GPQAStripGTRolloutProcessor(),
     aggregation_method="mean",
     passed_threshold=None,
-    num_runs=1,
+    num_runs=8,
     mode="pointwise",
 )
 def test_gpqa_pointwise(row: EvaluationRow) -> EvaluationRow:
diff --git a/eval_protocol/benchmarks/test_livebench_data_analysis.py b/eval_protocol/benchmarks/test_livebench_data_analysis.py
@@ -456,7 +456,7 @@ def test_livebench_cta_pointwise(row: EvaluationRow) -> EvaluationRow:
     rollout_processor=LiveBenchGroundTruthRolloutProcessor(_TABLEJOIN_ROWS),
     aggregation_method="mean",
     passed_threshold=None,
-    num_runs=1,
+    num_runs=4,
     mode="pointwise",
 )
 def test_livebench_tablejoin_pointwise(row: EvaluationRow) -> EvaluationRow:

Original file line number	Diff line number	Diff line change
`@@ -97,7 +97,7 @@ def __call__(self, rows: List[EvaluationRow], config: RolloutProcessorConfig) ->`
`97`	`97`	`rollout_processor=GPQAStripGTRolloutProcessor(),`
`98`	`98`	`aggregation_method="mean",`
`99`	`99`	`passed_threshold=None,`
`100`		`- num_runs=1,`
	`100`	`+ num_runs=8,`
`101`	`101`	`mode="pointwise",`
`102`	`102`	`)`
`103`	`103`	`def test_gpqa_pointwise(row: EvaluationRow) -> EvaluationRow:`
Original file line number	Diff line number	Diff line change
`@@ -456,7 +456,7 @@ def test_livebench_cta_pointwise(row: EvaluationRow) -> EvaluationRow:`
`456`	`456`	`rollout_processor=LiveBenchGroundTruthRolloutProcessor(_TABLEJOIN_ROWS),`
`457`	`457`	`aggregation_method="mean",`
`458`	`458`	`passed_threshold=None,`
`459`		`- num_runs=1,`
	`459`	`+ num_runs=4,`
`460`	`460`	`mode="pointwise",`
`461`	`461`	`)`
`462`	`462`	`def test_livebench_tablejoin_pointwise(row: EvaluationRow) -> EvaluationRow:`