Repository navigation
Expand file tree
/
Copy pathartifacts_codex_dry.json
More file actions
84 lines (84 loc) · 1.84 KB
/
Copy pathartifacts_codex_dry.json
File metadata and controls
84 lines (84 loc) · 1.84 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
{
"arms": {
"flat_n": {
"avg_candidates": 4.0,
"avg_oracle_calls": 4.0,
"n_tasks": 8,
"pass_rate": 1.0,
"receipt_reproducibility": 1.0
},
"frontier_codex-plan": {
"avg_candidates": 1.0,
"avg_oracle_calls": 1.0,
"n_tasks": 8,
"pass_rate": 1.0,
"receipt_reproducibility": 1.0
},
"local_ollama": {
"avg_candidates": 1.0,
"avg_oracle_calls": 1.0,
"n_tasks": 8,
"pass_rate": 0.0,
"receipt_reproducibility": 1.0
},
"no_search": {
"avg_candidates": 1.0,
"avg_oracle_calls": 1.0,
"n_tasks": 8,
"pass_rate": 1.0,
"receipt_reproducibility": 1.0
},
"single_shot": {
"avg_candidates": 1.0,
"avg_oracle_calls": 1.0,
"n_tasks": 8,
"pass_rate": 1.0,
"receipt_reproducibility": 1.0
},
"verified_inference": {
"avg_candidates": 4.0,
"avg_oracle_calls": 4.0,
"n_tasks": 8,
"pass_rate": 1.0,
"receipt_reproducibility": 1.0
}
},
"meta": {
"frontier_arms": [
"frontier_codex-plan"
],
"frontier_mode": "single",
"frontier_model_ref": "codex:gpt-5.3-codex-spark",
"frontier_model_refs": {
"frontier_codex-plan": "codex:gpt-5.3-codex-spark"
},
"local_arms": [
"local_ollama"
],
"local_mode": "sweep",
"local_model_refs": {
"local_ollama": "ollama:qwen2.5:7b"
},
"model_ref": "dry-run(reference)",
"n_tasks": 8,
"note": "frontier single-shot baseline comparison"
},
"metrics": {
"avg_oracle_calls": {
"good": "lower",
"max": null,
"min": 0.0
},
"pass_rate": {
"good": "higher",
"max": 1.0,
"min": 0.0
},
"receipt_reproducibility": {
"good": "higher",
"max": 1.0,
"min": 0.0
}
},
"schema": "m7-scorecard/v1"
}