| { |
| "label": "vl-a", |
| "backbone": "lfm2-vl", |
| "attention": "causal", |
| "calibration": { |
| "raw": { |
| "n": 1120, |
| "accuracy": 0.8848214285714285, |
| "nll": 0.8286656505531844, |
| "brier": 0.20143608167937646, |
| "ece": 0.09297216033922746, |
| "confident_error_rate": 0.07678571428571429, |
| "score_mae": 0.4503401108564636 |
| }, |
| "calibrated": { |
| "n": 1120, |
| "accuracy": 0.8848214285714285, |
| "nll": 0.3321787220851725, |
| "brier": 0.1661111770689924, |
| "ece": 0.02083344159724505, |
| "confident_error_rate": 0.015178571428571428, |
| "score_mae": 0.511491432084392 |
| }, |
| "temperature": 4.5947934199881395, |
| "by_source": { |
| "agnews": { |
| "n": 40, |
| "accuracy": 0.85, |
| "nll": 0.4613964179525684, |
| "brier": 0.23150962591792573, |
| "ece": 0.09537901553524014, |
| "confident_error_rate": 0.075 |
| }, |
| "agnews_yn": { |
| "n": 80, |
| "accuracy": 0.9375, |
| "nll": 0.18578394333721132, |
| "brier": 0.0901551691310664, |
| "ece": 0.030280076568509166, |
| "confident_error_rate": 0.025 |
| }, |
| "amazon": { |
| "n": 40, |
| "accuracy": 0.55, |
| "nll": 1.063923220607017, |
| "brier": 0.5988881204390519, |
| "ece": 0.14293638736581393, |
| "confident_error_rate": 0.025, |
| "score_mae": 0.5729569605099538 |
| }, |
| "banking77": { |
| "n": 40, |
| "accuracy": 0.675, |
| "nll": 1.5402265483550865, |
| "brier": 0.5359581656242952, |
| "ece": 0.2201891612204594, |
| "confident_error_rate": 0.0 |
| }, |
| "boolq": { |
| "n": 40, |
| "accuracy": 0.775, |
| "nll": 0.4769649130344238, |
| "brier": 0.31620540680281184, |
| "ece": 0.09690336717231163, |
| "confident_error_rate": 0.025 |
| }, |
| "dbpedia14": { |
| "n": 40, |
| "accuracy": 1.0, |
| "nll": 0.12480092447696775, |
| "brier": 0.048435455765173344, |
| "ece": 0.06948473292062886, |
| "confident_error_rate": 0.0 |
| }, |
| "imdb": { |
| "n": 40, |
| "accuracy": 0.875, |
| "nll": 0.2765451001786277, |
| "brier": 0.1442027427269653, |
| "ece": 0.0644099496689776, |
| "confident_error_rate": 0.025 |
| }, |
| "mnli": { |
| "n": 40, |
| "accuracy": 0.75, |
| "nll": 0.5184540211783258, |
| "brier": 0.3223852601810737, |
| "ece": 0.1619117743485849, |
| "confident_error_rate": 0.0 |
| }, |
| "sst5": { |
| "n": 40, |
| "accuracy": 0.5, |
| "nll": 1.2298145317333087, |
| "brier": 0.657291300878368, |
| "ece": 0.140258883357918, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.6459058613680704 |
| }, |
| "synthetic_approval_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.02713503023395747, |
| "brier": 0.012400901610244864, |
| "ece": 0.023288187036937025, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_approval_zh": { |
| "n": 13, |
| "accuracy": 0.9230769230769231, |
| "nll": 0.5102563080984519, |
| "brier": 0.15344398526112696, |
| "ece": 0.07446202415132108, |
| "confident_error_rate": 0.07692307692307693 |
| }, |
| "synthetic_deadline_en": { |
| "n": 12, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.2372234019882158, |
| "brier": 0.12317158447362637, |
| "ece": 0.10171254195717513, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.1977927131161766 |
| }, |
| "synthetic_deadline_zh": { |
| "n": 12, |
| "accuracy": 0.8333333333333334, |
| "nll": 0.2709692456015684, |
| "brier": 0.15859877650471121, |
| "ece": 0.18334363956648586, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.19353121308906318 |
| }, |
| "synthetic_membership_en": { |
| "n": 12, |
| "accuracy": 1.0, |
| "nll": 0.4864467399805366, |
| "brier": 0.2567791241792929, |
| "ece": 0.37032491734394646, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.38166059727082313 |
| }, |
| "synthetic_membership_zh": { |
| "n": 12, |
| "accuracy": 0.75, |
| "nll": 0.4737501553972594, |
| "brier": 0.2732967828693059, |
| "ece": 0.06227235426146951, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.2467490356606777 |
| }, |
| "synthetic_refund_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.00465089068399359, |
| "brier": 0.00010111006670935923, |
| "ece": 0.004625338642054944, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_refund_zh": { |
| "n": 13, |
| "accuracy": 0.9230769230769231, |
| "nll": 0.08045327525689482, |
| "brier": 0.048408113827161446, |
| "ece": 0.06068902817670701, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_snake_observable_teacher": { |
| "n": 500, |
| "accuracy": 0.996, |
| "nll": 0.02869776433137775, |
| "brier": 0.009226250091244111, |
| "ece": 0.02180775425742663, |
| "confident_error_rate": 0.0 |
| }, |
| "trec": { |
| "n": 40, |
| "accuracy": 0.85, |
| "nll": 0.5164713056059863, |
| "brier": 0.24584333997497865, |
| "ece": 0.08697971292754486, |
| "confident_error_rate": 0.075 |
| }, |
| "yelp": { |
| "n": 40, |
| "accuracy": 0.425, |
| "nll": 1.419889190687788, |
| "brier": 0.7752482539353915, |
| "ece": 0.2837610425917585, |
| "confident_error_rate": 0.075, |
| "score_mae": 0.6234811251354 |
| }, |
| "yelp_yn": { |
| "n": 40, |
| "accuracy": 0.9, |
| "nll": 0.29940020197899064, |
| "brier": 0.16628785487628298, |
| "ece": 0.07293930838007276, |
| "confident_error_rate": 0.05 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "n": 1070, |
| "accuracy": 0.8859813084112149, |
| "nll": 0.3321722535885867, |
| "brier": 0.1665772844080706, |
| "ece": 0.021660332496098857, |
| "confident_error_rate": 0.014953271028037384, |
| "score_mae": 0.5600499833693122 |
| }, |
| "zh": { |
| "n": 50, |
| "accuracy": 0.86, |
| "nll": 0.3323171479121088, |
| "brier": 0.15613648001271913, |
| "ece": 0.057765111700233476, |
| "confident_error_rate": 0.02, |
| "score_mae": 0.2201401243748704 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "n": 726, |
| "accuracy": 0.9476584022038568, |
| "nll": 0.20356603067047876, |
| "brier": 0.08558454886155338, |
| "ece": 0.021395347973956688, |
| "confident_error_rate": 0.009641873278236915 |
| }, |
| "noul": { |
| "n": 226, |
| "accuracy": 0.8938053097345132, |
| "nll": 0.25701539040659355, |
| "brier": 0.14562333459003557, |
| "ece": 0.03266069127275644, |
| "confident_error_rate": 0.02654867256637168 |
| }, |
| "score": { |
| "n": 168, |
| "accuracy": 0.6011904761904762, |
| "nll": 0.9890818585519972, |
| "brier": 0.5416622751573553, |
| "ece": 0.10589658337562606, |
| "confident_error_rate": 0.023809523809523808, |
| "score_mae": 0.511491432084392 |
| } |
| }, |
| "snake": { |
| "n": 500, |
| "accuracy": 0.996, |
| "nll": 0.02869776433137775, |
| "brier": 0.009226250091244111, |
| "ece": 0.02180775425742663, |
| "confident_error_rate": 0.0 |
| }, |
| "general": { |
| "n": 620, |
| "accuracy": 0.7951612903225806, |
| "nll": 0.576921429951136, |
| "brier": 0.2926312794704023, |
| "ece": 0.0518043898294689, |
| "confident_error_rate": 0.027419354838709678, |
| "score_mae": 0.511491432084392 |
| } |
| }, |
| "development": { |
| "raw": { |
| "n": 1380, |
| "accuracy": 0.8608695652173913, |
| "nll": 1.037557116940451, |
| "brier": 0.24184774106538062, |
| "ece": 0.11305555889037969, |
| "confident_error_rate": 0.09492753623188406, |
| "score_mae": 0.4030307561703975 |
| }, |
| "calibrated": { |
| "n": 1380, |
| "accuracy": 0.8608695652173913, |
| "nll": 0.3907753465146853, |
| "brier": 0.19660894207985088, |
| "ece": 0.0236135519422816, |
| "confident_error_rate": 0.018115942028985508, |
| "score_mae": 0.4692499422681266 |
| }, |
| "temperature": 4.5947934199881395, |
| "by_source": { |
| "agnews": { |
| "n": 60, |
| "accuracy": 0.8166666666666667, |
| "nll": 0.5432987762626141, |
| "brier": 0.2771836739526094, |
| "ece": 0.13067113910729217, |
| "confident_error_rate": 0.06666666666666667 |
| }, |
| "agnews_yn": { |
| "n": 120, |
| "accuracy": 0.9333333333333333, |
| "nll": 0.2474484161086575, |
| "brier": 0.12688493305603707, |
| "ece": 0.03359069154418503, |
| "confident_error_rate": 0.041666666666666664 |
| }, |
| "amazon": { |
| "n": 60, |
| "accuracy": 0.5833333333333334, |
| "nll": 1.019207187139469, |
| "brier": 0.5524721077332212, |
| "ece": 0.13899730979015376, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5781072710895353 |
| }, |
| "banking77": { |
| "n": 60, |
| "accuracy": 0.6833333333333333, |
| "nll": 1.386287052884947, |
| "brier": 0.4274681156509871, |
| "ece": 0.1742433197260666, |
| "confident_error_rate": 0.0 |
| }, |
| "boolq": { |
| "n": 60, |
| "accuracy": 0.6333333333333333, |
| "nll": 0.7842629466005272, |
| "brier": 0.5372472386575228, |
| "ece": 0.21517449578016706, |
| "confident_error_rate": 0.06666666666666667 |
| }, |
| "dbpedia14": { |
| "n": 60, |
| "accuracy": 0.9666666666666667, |
| "nll": 0.17627788368667227, |
| "brier": 0.05372020233696572, |
| "ece": 0.027486791887844184, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "imdb": { |
| "n": 60, |
| "accuracy": 0.8, |
| "nll": 0.5042462474442398, |
| "brier": 0.27497683228600267, |
| "ece": 0.11728494718294821, |
| "confident_error_rate": 0.06666666666666667 |
| }, |
| "mnli": { |
| "n": 60, |
| "accuracy": 0.6333333333333333, |
| "nll": 0.8209178273118785, |
| "brier": 0.47354704169414424, |
| "ece": 0.1454240547092434, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "sst5": { |
| "n": 60, |
| "accuracy": 0.5, |
| "nll": 1.09419991411571, |
| "brier": 0.6078575013753795, |
| "ece": 0.10528637320636364, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.5626427438598316 |
| }, |
| "synthetic_approval_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.00305151055912005, |
| "brier": 9.100589788026809e-05, |
| "ece": 0.003028460775100883, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_approval_zh": { |
| "n": 13, |
| "accuracy": 0.9230769230769231, |
| "nll": 0.31252450790157693, |
| "brier": 0.14840695576773935, |
| "ece": 0.07158792870719877, |
| "confident_error_rate": 0.07692307692307693 |
| }, |
| "synthetic_deadline_en": { |
| "n": 12, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.22089136851581057, |
| "brier": 0.10851671633317528, |
| "ece": 0.09197092561596329, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.17273660444065683 |
| }, |
| "synthetic_deadline_zh": { |
| "n": 12, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.22471943817596504, |
| "brier": 0.09609427381476522, |
| "ece": 0.14289170677922886, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.12739659503185494 |
| }, |
| "synthetic_membership_en": { |
| "n": 12, |
| "accuracy": 0.8333333333333334, |
| "nll": 0.5289100636414515, |
| "brier": 0.3247949977066678, |
| "ece": 0.3540032822410373, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.3658395223985365 |
| }, |
| "synthetic_membership_zh": { |
| "n": 12, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.3384060300915314, |
| "brier": 0.17768819657830845, |
| "ece": 0.1240372424832984, |
| "confident_error_rate": 0.0, |
| "score_mae": 0.14854067532162193 |
| }, |
| "synthetic_refund_en": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.004772849934686132, |
| "brier": 9.814589364790191e-05, |
| "ece": 0.004748060980601321, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_refund_zh": { |
| "n": 13, |
| "accuracy": 1.0, |
| "nll": 0.026827711733167094, |
| "brier": 0.002599816753125005, |
| "ece": 0.02614938190979532, |
| "confident_error_rate": 0.0 |
| }, |
| "synthetic_snake_observable_teacher": { |
| "n": 500, |
| "accuracy": 0.988, |
| "nll": 0.03911276666514134, |
| "brier": 0.015846521973586804, |
| "ece": 0.022786703593587133, |
| "confident_error_rate": 0.0 |
| }, |
| "trec": { |
| "n": 60, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.3414991446020253, |
| "brier": 0.13705633897022337, |
| "ece": 0.08460856930544054, |
| "confident_error_rate": 0.016666666666666666 |
| }, |
| "yelp": { |
| "n": 60, |
| "accuracy": 0.6833333333333333, |
| "nll": 0.9475770833309305, |
| "brier": 0.4856780158774344, |
| "ece": 0.09408258083704575, |
| "confident_error_rate": 0.03333333333333333, |
| "score_mae": 0.47949708623098025 |
| }, |
| "yelp_yn": { |
| "n": 60, |
| "accuracy": 0.9166666666666666, |
| "nll": 0.21141537958578643, |
| "brier": 0.13479642958918142, |
| "ece": 0.05396240895190205, |
| "confident_error_rate": 0.03333333333333333 |
| } |
| }, |
| "by_language": { |
| "en": { |
| "n": 1330, |
| "accuracy": 0.8578947368421053, |
| "nll": 0.3970683411397024, |
| "brier": 0.20005403186669637, |
| "ece": 0.026465524784310385, |
| "confident_error_rate": 0.01804511278195489, |
| "score_mae": 0.5082242136906429 |
| }, |
| "zh": { |
| "n": 50, |
| "accuracy": 0.94, |
| "nll": 0.22338168948923257, |
| "brier": 0.10496955374976241, |
| "ece": 0.05224892565568175, |
| "confident_error_rate": 0.02, |
| "score_mae": 0.13796863517673844 |
| } |
| }, |
| "by_qtype": { |
| "choice": { |
| "n": 826, |
| "accuracy": 0.9200968523002422, |
| "nll": 0.26604807827778204, |
| "brier": 0.11137077099847731, |
| "ece": 0.01219164077100229, |
| "confident_error_rate": 0.009685230024213076 |
| }, |
| "noul": { |
| "n": 326, |
| "accuracy": 0.8558282208588958, |
| "nll": 0.3684051891182644, |
| "brier": 0.2211119494266715, |
| "ece": 0.074143327637379, |
| "confident_error_rate": 0.046012269938650305 |
| }, |
| "score": { |
| "n": 228, |
| "accuracy": 0.6535087719298246, |
| "nll": 0.8746235696502792, |
| "brier": 0.4703753847033207, |
| "ece": 0.027926810584187323, |
| "confident_error_rate": 0.008771929824561403, |
| "score_mae": 0.4692499422681266 |
| } |
| }, |
| "snake": { |
| "n": 500, |
| "accuracy": 0.988, |
| "nll": 0.03911276666514134, |
| "brier": 0.015846521973586804, |
| "ece": 0.022786703593587133, |
| "confident_error_rate": 0.0 |
| }, |
| "general": { |
| "n": 880, |
| "accuracy": 0.7886363636363637, |
| "nll": 0.5905836305201081, |
| "brier": 0.2993148625947737, |
| "ece": 0.04887231156165163, |
| "confident_error_rate": 0.028409090909090908, |
| "score_mae": 0.4692499422681266 |
| } |
| }, |
| "training_seconds": 1089.3942992687225, |
| "optimizer_steps": 4476, |
| "forward_tokens": 22269908, |
| "snake_closed_loop": { |
| "complete": true, |
| "food": { |
| "count": 20, |
| "mean": 41.4, |
| "p50": 41.5, |
| "p95": 50, |
| "max": 53 |
| }, |
| "collisions": 0, |
| "terminal_reasons": { |
| "max_steps": 20 |
| }, |
| "prediction_call_ms": { |
| "count": 1500, |
| "mean": 47.461315778669665, |
| "p50": 48.8280649999524, |
| "p95": 51.10321799998019, |
| "max": 57.466298000008464 |
| } |
| } |
| } |
|
|
