| { | |
| "model": "mvbalaji/od1-typed-decisions", | |
| "dataset": "LocalLLaMA/typed-decisions@c76749ec58bd8c3d2ea706b31c333a9059c38f90", | |
| "split": "test (400 cases / 2,000 decisions)", | |
| "mode": "specialist (fitted on the train split)", | |
| "card_metrics": { | |
| "acc": 0.7965, | |
| "soft_acc": 0.521672958018039, | |
| "kl": 0.08194745417901107, | |
| "tv": 0.13020994535529862, | |
| "brier": 0.04549292501917222, | |
| "ece": 0.15645043126221042, | |
| "macro_f1": 0.672337649306128, | |
| "score_mae": 0.20133834253665778, | |
| "within_1": 0.985, | |
| "acc_by_type": { | |
| "choice": 0.7683333333333333, | |
| "noul": 0.8666666666666667, | |
| "score": 0.765 | |
| }, | |
| "n": 2000, | |
| "kind": "specialist", | |
| "mean_na_mass_removed": 0.00012872512336616762 | |
| }, | |
| "formula_check_vs_card_reference_rows": { | |
| "Uniform": { | |
| "acc": { | |
| "ours": 0.274, | |
| "card": 0.308 | |
| }, | |
| "soft_acc": { | |
| "ours": 0.318, | |
| "card": 0.311 | |
| }, | |
| "macro_f1": { | |
| "ours": 0.141, | |
| "card": 0.152 | |
| }, | |
| "kl": { | |
| "ours": 0.444, | |
| "card": 0.444 | |
| }, | |
| "tv": { | |
| "ours": 0.381, | |
| "card": 0.381 | |
| }, | |
| "brier": { | |
| "ours": 0.238, | |
| "card": 0.238 | |
| }, | |
| "ece": { | |
| "ours": 0.048, | |
| "card": 0.169 | |
| } | |
| }, | |
| "Prior": { | |
| "acc": { | |
| "ours": 0.483, | |
| "card": 0.47 | |
| }, | |
| "soft_acc": { | |
| "ours": 0.39, | |
| "card": 0.43 | |
| }, | |
| "macro_f1": { | |
| "ours": 0.212, | |
| "card": 0.207 | |
| }, | |
| "kl": { | |
| "ours": 0.394, | |
| "card": 0.347 | |
| }, | |
| "tv": { | |
| "ours": 0.316, | |
| "card": 0.317 | |
| }, | |
| "brier": { | |
| "ours": 0.197, | |
| "card": 0.189 | |
| }, | |
| "ece": { | |
| "ours": 0.039, | |
| "card": 0.088 | |
| } | |
| } | |
| }, | |
| "overlap_subsets": { | |
| "full": { | |
| "correct": 1593, | |
| "total": 2000, | |
| "accuracy": 0.7965, | |
| "states": 400 | |
| }, | |
| "template_overlap": { | |
| "correct": 336, | |
| "total": 470, | |
| "accuracy": 0.7148936170212766, | |
| "states": 94 | |
| }, | |
| "lower_overlap": { | |
| "correct": 1257, | |
| "total": 1530, | |
| "accuracy": 0.8215686274509804, | |
| "states": 306 | |
| }, | |
| "near_duplicate": { | |
| "correct": 95, | |
| "total": 130, | |
| "accuracy": 0.7307692307692307, | |
| "states": 26 | |
| } | |
| }, | |
| "corrected_input_variant_A1_not_released": { | |
| "test": { | |
| "correct": 1606, | |
| "total": 2000, | |
| "accuracy": 0.803 | |
| }, | |
| "paired_diff_vs_released": { | |
| "delta": 0.0065, | |
| "ci95": [ | |
| -0.0045, | |
| 0.0165 | |
| ], | |
| "states": 400 | |
| }, | |
| "fresh_teacher_labelled_set": { | |
| "released": { | |
| "correct": 917, | |
| "total": 1180, | |
| "accuracy": 0.7771186440677966 | |
| }, | |
| "a1": { | |
| "correct": 902, | |
| "total": 1180, | |
| "accuracy": 0.764406779661017 | |
| }, | |
| "paired_diff": { | |
| "delta": -0.012711864406779662, | |
| "ci95": [ | |
| -0.026271186440677965, | |
| 0.0 | |
| ], | |
| "states": 236 | |
| } | |
| } | |
| } | |
| } |