| {"record_id": "jevbench:original-policy-01-0", "group_id": "jevbench:original-policy-01", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9838292002677917, "true": 0.016170842573046684}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9838292002677917, "record_seconds": 1.375101308338344, "batch_seconds": 1.375101308338344, "batch_records": 1, "timing_scope": "record", "brier": 0.0005229909438639879, "nll": 0.01630297303199768} |
| {"record_id": "jevbench:original-policy-01-1", "group_id": "jevbench:original-policy-01", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9819019436836243, "true": 0.01809808984398842}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9819019436836243, "record_seconds": 0.09580718539655209, "batch_seconds": 0.09580718539655209, "batch_records": 1, "timing_scope": "record", "brier": 0.0006550804828293622, "nll": 0.01826382987201214} |
| {"record_id": "jevbench:original-policy-02-0", "group_id": "jevbench:original-policy-02", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.01824221946299076, "true": 0.9817578196525574}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9817578196525574, "record_seconds": 0.09445640165358782, "batch_seconds": 0.09445640165358782, "batch_records": 1, "timing_scope": "record", "brier": 0.0006655557081103325, "nll": 0.01841062121093273} |
| {"record_id": "jevbench:original-policy-02-1", "group_id": "jevbench:original-policy-02", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.03165252506732941, "true": 0.968347430229187}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.968347430229187, "record_seconds": 0.09289274457842112, "batch_seconds": 0.09289274457842112, "batch_records": 1, "timing_scope": "record", "brier": 0.002003767527639866, "nll": 0.03216433897614479} |
| {"record_id": "jevbench:original-policy-03-0", "group_id": "jevbench:original-policy-03", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.3640267550945282, "true": 0.6359732747077942}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.6359732747077942, "record_seconds": 0.09221394266933203, "batch_seconds": 0.09221394266933203, "batch_records": 1, "timing_scope": "record", "brier": 0.2650309205055237, "nll": 0.4525987505912781} |
| {"record_id": "jevbench:original-policy-03-1", "group_id": "jevbench:original-policy-03", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.10579399764537811, "true": 0.8942059874534607}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.8942059874534607, "record_seconds": 0.09248625580221415, "batch_seconds": 0.09248625580221415, "batch_records": 1, "timing_scope": "record", "brier": 0.02238474413752556, "nll": 0.11181911826133728} |
| {"record_id": "jevbench:original-policy-04-0", "group_id": "jevbench:original-policy-04", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9715238213539124, "true": 0.028476174920797348}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9715238213539124, "record_seconds": 0.09162857290357351, "batch_seconds": 0.09162857290357351, "batch_records": 1, "timing_scope": "record", "brier": 0.0016217853408306837, "nll": 0.02888949029147625} |
| {"record_id": "jevbench:original-policy-04-1", "group_id": "jevbench:original-policy-04", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9828407764434814, "true": 0.017159167677164078}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9828407764434814, "record_seconds": 0.09205986373126507, "batch_seconds": 0.09205986373126507, "batch_records": 1, "timing_scope": "record", "brier": 0.000588875962421298, "nll": 0.017308149486780167} |
| {"record_id": "jevbench:original-policy-05-0", "group_id": "jevbench:original-policy-05", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9518351554870605, "true": 0.04816482216119766}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9518351554870605, "record_seconds": 0.09233530703932047, "batch_seconds": 0.09233530703932047, "batch_records": 1, "timing_scope": "record", "brier": 0.004639702383428812, "nll": 0.04936341568827629} |
| {"record_id": "jevbench:original-policy-05-1", "group_id": "jevbench:original-policy-05", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9664505124092102, "true": 0.03354942426085472}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9664505124092102, "record_seconds": 0.09231139067560434, "batch_seconds": 0.09231139067560434, "batch_records": 1, "timing_scope": "record", "brier": 0.0022511319257318974, "nll": 0.034125182777643204} |
| {"record_id": "jevbench:original-policy-06-0", "group_id": "jevbench:original-policy-06", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.01661483198404312, "true": 0.9833852052688599}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9833852052688599, "record_seconds": 0.09181586746126413, "batch_seconds": 0.09181586746126413, "batch_records": 1, "timing_scope": "record", "brier": 0.0005521040875464678, "nll": 0.01675436832010746} |
| {"record_id": "jevbench:original-policy-06-1", "group_id": "jevbench:original-policy-06", "question_id": "decision", "source": "jevbench/original/policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.38817593455314636, "true": 0.611824095249176}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.611824095249176, "record_seconds": 0.09181506838649511, "batch_seconds": 0.09181506838649511, "batch_records": 1, "timing_scope": "record", "brier": 0.301361083984375, "nll": 0.4913104772567749} |
| {"record_id": "jevbench:original-intent-01-0", "group_id": "jevbench:original-intent-01", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "refund", "probabilities": {"cancel": 0.04920588433742523, "change_address": 0.006849173456430435, "other": 0.1541178673505783, "refund": 0.7850407361984253, "status": 0.004786288365721703}, "none_candidate_id": null, "predicted_none": false, "label": "refund", "correct": true, "confidence": 0.7850407361984253, "record_seconds": 0.09499531891196966, "batch_seconds": 0.09499531891196966, "batch_records": 1, "timing_scope": "record", "brier": 0.07245083898305893, "nll": 0.2420196682214737} |
| {"record_id": "jevbench:original-intent-01-1", "group_id": "jevbench:original-intent-01", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "refund", "probabilities": {"cancel": 0.04902530834078789, "change_address": 0.007075275294482708, "other": 0.1975967288017273, "refund": 0.7404109239578247, "status": 0.005891779903322458}, "none_candidate_id": null, "predicted_none": false, "label": "refund", "correct": true, "confidence": 0.7404109239578247, "record_seconds": 0.09277319628745317, "batch_seconds": 0.09277319628745317, "batch_records": 1, "timing_scope": "record", "brier": 0.10891920328140259, "nll": 0.3005499541759491} |
| {"record_id": "jevbench:original-intent-02-0", "group_id": "jevbench:original-intent-02", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "other", "probabilities": {"cancel": 0.007295570336282253, "change_address": 0.07357282936573029, "other": 0.6553636193275452, "refund": 0.0055071404203772545, "status": 0.2582608461380005}, "none_candidate_id": null, "predicted_none": false, "label": "status", "correct": false, "confidence": 0.6553636193275452, "record_seconds": 0.09152832254767418, "batch_seconds": 0.09152832254767418, "batch_records": 1, "timing_scope": "record", "brier": 0.9851749539375305, "nll": 1.3537851572036743} |
| {"record_id": "jevbench:original-intent-02-1", "group_id": "jevbench:original-intent-02", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "status", "probabilities": {"cancel": 0.006421714089810848, "change_address": 0.007567776367068291, "other": 0.05151044949889183, "refund": 0.005047861952334642, "status": 0.9294521808624268}, "none_candidate_id": null, "predicted_none": false, "label": "status", "correct": true, "confidence": 0.9294521808624268, "record_seconds": 0.0917139770463109, "batch_seconds": 0.0917139770463109, "batch_records": 1, "timing_scope": "record", "brier": 0.0077543118968605995, "nll": 0.07315991818904877} |
| {"record_id": "jevbench:original-intent-03-0", "group_id": "jevbench:original-intent-03", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "change_address", "probabilities": {"cancel": 0.006300788838416338, "change_address": 0.9390687346458435, "other": 0.03805028647184372, "refund": 0.007017828989773989, "status": 0.009562362916767597}, "none_candidate_id": null, "predicted_none": false, "label": "change_address", "correct": true, "confidence": 0.9390687346458435, "record_seconds": 0.09229207504540682, "batch_seconds": 0.09229207504540682, "batch_records": 1, "timing_scope": "record", "brier": 0.005340831819921732, "nll": 0.06286660581827164} |
| {"record_id": "jevbench:original-intent-03-1", "group_id": "jevbench:original-intent-03", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "change_address", "probabilities": {"cancel": 0.003640597453340888, "change_address": 0.9648473858833313, "other": 0.023510431870818138, "refund": 0.003552419366315007, "status": 0.004449078813195229}, "none_candidate_id": null, "predicted_none": false, "label": "change_address", "correct": true, "confidence": 0.9648473858833313, "record_seconds": 0.09363086055964231, "batch_seconds": 0.09363086055964231, "batch_records": 1, "timing_scope": "record", "brier": 0.0018341145478188992, "nll": 0.03578533977270126} |
| {"record_id": "jevbench:original-intent-04-0", "group_id": "jevbench:original-intent-04", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "cancel", "probabilities": {"cancel": 0.8397079110145569, "change_address": 0.007443739101290703, "other": 0.12877397239208221, "refund": 0.01847805827856064, "status": 0.005596349481493235}, "none_candidate_id": null, "predicted_none": false, "label": "cancel", "correct": true, "confidence": 0.8397079110145569, "record_seconds": 0.09500238578766584, "batch_seconds": 0.09500238578766584, "batch_records": 1, "timing_scope": "record", "brier": 0.04270445555448532, "nll": 0.17470116913318634} |
| {"record_id": "jevbench:original-intent-04-1", "group_id": "jevbench:original-intent-04", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "cancel", "probabilities": {"cancel": 0.901157557964325, "change_address": 0.006284218281507492, "other": 0.07751098275184631, "refund": 0.009882895275950432, "status": 0.005164295434951782}, "none_candidate_id": null, "predicted_none": false, "label": "cancel", "correct": true, "confidence": 0.901157557964325, "record_seconds": 0.09359128307551146, "batch_seconds": 0.09359128307551146, "batch_records": 1, "timing_scope": "record", "brier": 0.015941614285111427, "nll": 0.10407516360282898} |
| {"record_id": "jevbench:original-intent-05-0", "group_id": "jevbench:original-intent-05", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "other", "probabilities": {"cancel": 0.014578565955162048, "change_address": 0.008526241406798363, "other": 0.8751257658004761, "refund": 0.09244894981384277, "status": 0.009320506826043129}, "none_candidate_id": null, "predicted_none": false, "label": "other", "correct": true, "confidence": 0.8751257658004761, "record_seconds": 0.09223337657749653, "batch_seconds": 0.09223337657749653, "batch_records": 1, "timing_scope": "record", "brier": 0.024512486532330513, "nll": 0.13338766992092133} |
| {"record_id": "jevbench:original-intent-05-1", "group_id": "jevbench:original-intent-05", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "other", "probabilities": {"cancel": 0.028470907360315323, "change_address": 0.008978397585451603, "other": 0.8561321496963501, "refund": 0.0964384451508522, "status": 0.009980084374547005}, "none_candidate_id": null, "predicted_none": false, "label": "other", "correct": true, "confidence": 0.8561321496963501, "record_seconds": 0.0926353158429265, "batch_seconds": 0.0926353158429265, "batch_records": 1, "timing_scope": "record", "brier": 0.030989138409495354, "nll": 0.15533053874969482} |
| {"record_id": "jevbench:original-intent-06-0", "group_id": "jevbench:original-intent-06", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "status", "probabilities": {"cancel": 0.03532528504729271, "change_address": 0.011102276854217052, "other": 0.13720235228538513, "refund": 0.016235370188951492, "status": 0.8001346588134766}, "none_candidate_id": null, "predicted_none": false, "label": "status", "correct": true, "confidence": 0.8001346588134766, "record_seconds": 0.09230886679142714, "batch_seconds": 0.09230886679142714, "batch_records": 1, "timing_scope": "record", "brier": 0.060405366122722626, "nll": 0.2229752391576767} |
| {"record_id": "jevbench:original-intent-06-1", "group_id": "jevbench:original-intent-06", "question_id": "decision", "source": "jevbench/original/intent", "type": "choice", "prediction": "status", "probabilities": {"cancel": 0.009324096143245697, "change_address": 0.008825082331895828, "other": 0.08252221345901489, "refund": 0.010529634542763233, "status": 0.8887990117073059}, "none_candidate_id": null, "predicted_none": false, "label": "status", "correct": true, "confidence": 0.8887990117073059, "record_seconds": 0.09188603423535824, "batch_seconds": 0.09188603423535824, "batch_records": 1, "timing_scope": "record", "brier": 0.019451269879937172, "nll": 0.11788415163755417} |
| {"record_id": "jevbench:original-ordinal-01-0", "group_id": "jevbench:original-ordinal-01", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "0", "probabilities": {"0": 0.8964240550994873, "1": 0.08879097551107407, "2": 0.0096696512773633, "3": 0.005115325562655926}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.8964240550994873, "record_seconds": 0.09495967160910368, "batch_seconds": 0.09495967160910368, "batch_records": 1, "timing_scope": "record", "brier": 0.018731484189629555, "nll": 0.10934170335531235} |
| {"record_id": "jevbench:original-ordinal-01-1", "group_id": "jevbench:original-ordinal-01", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "0", "probabilities": {"0": 0.9133036136627197, "1": 0.07391936331987381, "2": 0.00819550734013319, "3": 0.004581550136208534}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.9133036136627197, "record_seconds": 0.09250781126320362, "batch_seconds": 0.09250781126320362, "batch_records": 1, "timing_scope": "record", "brier": 0.013068492524325848, "nll": 0.09068690985441208} |
| {"record_id": "jevbench:original-ordinal-02-0", "group_id": "jevbench:original-ordinal-02", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "1", "probabilities": {"0": 0.09265924245119095, "1": 0.8221272230148315, "2": 0.07799921929836273, "3": 0.007214277051389217}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": true, "confidence": 0.8221272230148315, "record_seconds": 0.09219043701887131, "batch_seconds": 0.09219043701887131, "batch_records": 1, "timing_scope": "record", "brier": 0.04636038839817047, "nll": 0.1958601176738739} |
| {"record_id": "jevbench:original-ordinal-02-1", "group_id": "jevbench:original-ordinal-02", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "1", "probabilities": {"0": 0.05497284233570099, "1": 0.9235140681266785, "2": 0.014205737970769405, "3": 0.007307275664061308}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": true, "confidence": 0.9235140681266785, "record_seconds": 0.0928940512239933, "batch_seconds": 0.0928940512239933, "batch_records": 1, "timing_scope": "record", "brier": 0.00912731047719717, "nll": 0.0795692428946495} |
| {"record_id": "jevbench:original-ordinal-03-0", "group_id": "jevbench:original-ordinal-03", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "2", "probabilities": {"0": 0.043039608746767044, "1": 0.04582200571894646, "2": 0.9056392908096313, "3": 0.005499167367815971}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.9056392908096313, "record_seconds": 0.09541366808116436, "batch_seconds": 0.09541366808116436, "batch_records": 1, "timing_scope": "record", "brier": 0.012886247597634792, "nll": 0.09911418706178665} |
| {"record_id": "jevbench:original-ordinal-03-1", "group_id": "jevbench:original-ordinal-03", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "2", "probabilities": {"0": 0.04011976346373558, "1": 0.044318050146102905, "2": 0.9071323871612549, "3": 0.00842980295419693}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.9071323871612549, "record_seconds": 0.09448737185448408, "batch_seconds": 0.09448737185448408, "batch_records": 1, "timing_scope": "record", "brier": 0.012269140221178532, "nll": 0.09746687859296799} |
| {"record_id": "jevbench:original-ordinal-04-0", "group_id": "jevbench:original-ordinal-04", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "3", "probabilities": {"0": 0.017748603597283363, "1": 0.025890707969665527, "2": 0.01854104734957218, "3": 0.9378196001052856}, "none_candidate_id": null, "predicted_none": false, "label": "3", "correct": true, "confidence": 0.9378196001052856, "record_seconds": 0.09334637131541967, "batch_seconds": 0.09334637131541967, "batch_records": 1, "timing_scope": "record", "brier": 0.005195514298975468, "nll": 0.06419767439365387} |
| {"record_id": "jevbench:original-ordinal-04-1", "group_id": "jevbench:original-ordinal-04", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "3", "probabilities": {"0": 0.023318978026509285, "1": 0.03051363304257393, "2": 0.02012704126536846, "3": 0.9260403513908386}, "none_candidate_id": null, "predicted_none": false, "label": "3", "correct": true, "confidence": 0.9260403513908386, "record_seconds": 0.09679166693240404, "batch_seconds": 0.09679166693240404, "batch_records": 1, "timing_scope": "record", "brier": 0.007349983789026737, "nll": 0.07683747261762619} |
| {"record_id": "jevbench:original-ordinal-05-0", "group_id": "jevbench:original-ordinal-05", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "0", "probabilities": {"0": 0.7995295524597168, "1": 0.16939236223697662, "2": 0.021347254514694214, "3": 0.009730815887451172}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.7995295524597168, "record_seconds": 0.09583998750895262, "batch_seconds": 0.09583998750895262, "batch_records": 1, "timing_scope": "record", "brier": 0.0694325715303421, "nll": 0.22373178601264954} |
| {"record_id": "jevbench:original-ordinal-05-1", "group_id": "jevbench:original-ordinal-05", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "0", "probabilities": {"0": 0.872722864151001, "1": 0.09409326314926147, "2": 0.021700507029891014, "3": 0.011483338661491871}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.872722864151001, "record_seconds": 0.09302619099617004, "batch_seconds": 0.09302619099617004, "batch_records": 1, "timing_scope": "record", "brier": 0.02565579302608967, "nll": 0.1361372321844101} |
| {"record_id": "jevbench:original-ordinal-06-0", "group_id": "jevbench:original-ordinal-06", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "2", "probabilities": {"0": 0.03979723900556564, "1": 0.057577501982450485, "2": 0.8988871574401855, "3": 0.003738199360668659}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.8988871574401855, "record_seconds": 0.0935876751318574, "batch_seconds": 0.0935876751318574, "batch_records": 1, "timing_scope": "record", "brier": 0.015136769972741604, "nll": 0.10659777373075485} |
| {"record_id": "jevbench:original-ordinal-06-1", "group_id": "jevbench:original-ordinal-06", "question_id": "decision", "source": "jevbench/original/ordinal", "type": "score", "prediction": "2", "probabilities": {"0": 0.016217591241002083, "1": 0.022123318165540695, "2": 0.9578564763069153, "3": 0.0038026333786547184}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.9578564763069153, "record_seconds": 0.09480876475572586, "batch_seconds": 0.09480876475572586, "batch_records": 1, "timing_scope": "record", "brier": 0.0025429881643503904, "nll": 0.04305732995271683} |
| {"record_id": "jevbench:original-extraction-01-0", "group_id": "jevbench:original-extraction-01", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "pickup", "probabilities": {"courier": 0.012699749320745468, "pickup": 0.9376670122146606, "post": 0.01990526169538498, "unknown": 0.029727937653660774}, "none_candidate_id": null, "predicted_none": false, "label": "pickup", "correct": true, "confidence": 0.9376670122146606, "record_seconds": 0.09301763121038675, "batch_seconds": 0.09301763121038675, "batch_records": 1, "timing_scope": "record", "brier": 0.005326654762029648, "nll": 0.06436038762331009} |
| {"record_id": "jevbench:original-extraction-01-1", "group_id": "jevbench:original-extraction-01", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "pickup", "probabilities": {"courier": 0.031132524833083153, "pickup": 0.5601819157600403, "post": 0.1582348346710205, "unknown": 0.2504507005214691}, "none_candidate_id": null, "predicted_none": false, "label": "pickup", "correct": true, "confidence": 0.5601819157600403, "record_seconds": 0.0929694278165698, "batch_seconds": 0.0929694278165698, "batch_records": 1, "timing_scope": "record", "brier": 0.2821729779243469, "nll": 0.5794937014579773} |
| {"record_id": "jevbench:original-extraction-02-0", "group_id": "jevbench:original-extraction-02", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "unknown", "probabilities": {"courier": 0.014672712422907352, "pickup": 0.014892463572323322, "post": 0.08467475324869156, "unknown": 0.8857601284980774}, "none_candidate_id": null, "predicted_none": false, "label": "unknown", "correct": true, "confidence": 0.8857601284980774, "record_seconds": 0.09208492655307055, "batch_seconds": 0.09208492655307055, "batch_records": 1, "timing_scope": "record", "brier": 0.02065763622522354, "nll": 0.12130910158157349} |
| {"record_id": "jevbench:original-extraction-02-1", "group_id": "jevbench:original-extraction-02", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "unknown", "probabilities": {"courier": 0.009442984126508236, "pickup": 0.023871861398220062, "post": 0.010268197394907475, "unknown": 0.9564169645309448}, "none_candidate_id": null, "predicted_none": false, "label": "unknown", "correct": true, "confidence": 0.9564169645309448, "record_seconds": 0.09207561612129211, "batch_seconds": 0.09207561612129211, "batch_records": 1, "timing_scope": "record", "brier": 0.0026639525312930346, "nll": 0.04456130415201187} |
| {"record_id": "jevbench:original-extraction-03-0", "group_id": "jevbench:original-extraction-03", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "courier", "probabilities": {"courier": 0.8942757844924927, "pickup": 0.04201977327466011, "post": 0.027389945462346077, "unknown": 0.036314480006694794}, "none_candidate_id": null, "predicted_none": false, "label": "courier", "correct": true, "confidence": 0.8942757844924927, "record_seconds": 0.09235639404505491, "batch_seconds": 0.09235639404505491, "batch_records": 1, "timing_scope": "record", "brier": 0.015012221410870552, "nll": 0.1117410659790039} |
| {"record_id": "jevbench:original-extraction-03-1", "group_id": "jevbench:original-extraction-03", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "courier", "probabilities": {"courier": 0.9470943808555603, "pickup": 0.014381698332726955, "post": 0.02219625562429428, "unknown": 0.016327688470482826}, "none_candidate_id": null, "predicted_none": false, "label": "courier", "correct": true, "confidence": 0.9470943808555603, "record_seconds": 0.09392414055764675, "batch_seconds": 0.09392414055764675, "batch_records": 1, "timing_scope": "record", "brier": 0.0037651050370186567, "nll": 0.05435652658343315} |
| {"record_id": "jevbench:original-extraction-04-0", "group_id": "jevbench:original-extraction-04", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "post", "probabilities": {"courier": 0.009538297541439533, "pickup": 0.008565046824514866, "post": 0.9577484130859375, "unknown": 0.02414819970726967}, "none_candidate_id": null, "predicted_none": false, "label": "post", "correct": true, "confidence": 0.9577484130859375, "record_seconds": 0.09308502916246653, "batch_seconds": 0.09308502916246653, "batch_records": 1, "timing_scope": "record", "brier": 0.002532671205699444, "nll": 0.043170154094696045} |
| {"record_id": "jevbench:original-extraction-04-1", "group_id": "jevbench:original-extraction-04", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "post", "probabilities": {"courier": 0.007560821250081062, "pickup": 0.0036303175147622824, "post": 0.9806631207466125, "unknown": 0.00814575981348753}, "none_candidate_id": null, "predicted_none": false, "label": "post", "correct": true, "confidence": 0.9806631207466125, "record_seconds": 0.09219395648688078, "batch_seconds": 0.09219395648688078, "batch_records": 1, "timing_scope": "record", "brier": 0.0005106135504320264, "nll": 0.019526282325387} |
| {"record_id": "jevbench:original-extraction-05-0", "group_id": "jevbench:original-extraction-05", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "unknown", "probabilities": {"courier": 0.012589944526553154, "pickup": 0.01775483600795269, "post": 0.01650083437561989, "unknown": 0.9531543850898743}, "none_candidate_id": null, "predicted_none": false, "label": "unknown", "correct": true, "confidence": 0.9531543850898743, "record_seconds": 0.09229068644344807, "batch_seconds": 0.09229068644344807, "batch_records": 1, "timing_scope": "record", "brier": 0.002940530190244317, "nll": 0.047978390008211136} |
| {"record_id": "jevbench:original-extraction-05-1", "group_id": "jevbench:original-extraction-05", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "unknown", "probabilities": {"courier": 0.03149859607219696, "pickup": 0.042276833206415176, "post": 0.046400610357522964, "unknown": 0.8798239827156067}, "none_candidate_id": null, "predicted_none": false, "label": "unknown", "correct": true, "confidence": 0.8798239827156067, "record_seconds": 0.09218011610209942, "batch_seconds": 0.09218011610209942, "batch_records": 1, "timing_scope": "record", "brier": 0.0193747840821743, "nll": 0.12803341448307037} |
| {"record_id": "jevbench:original-extraction-06-0", "group_id": "jevbench:original-extraction-06", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "courier", "probabilities": {"courier": 0.9890323281288147, "pickup": 0.0033821286633610725, "post": 0.0035310443490743637, "unknown": 0.004054527264088392}, "none_candidate_id": null, "predicted_none": false, "label": "courier", "correct": true, "confidence": 0.9890323281288147, "record_seconds": 0.09347787965089083, "batch_seconds": 0.09347787965089083, "batch_records": 1, "timing_scope": "record", "brier": 0.0001606360892765224, "nll": 0.011028259992599487} |
| {"record_id": "jevbench:original-extraction-06-1", "group_id": "jevbench:original-extraction-06", "question_id": "decision", "source": "jevbench/original/extraction", "type": "choice", "prediction": "courier", "probabilities": {"courier": 0.9713413715362549, "pickup": 0.008360929787158966, "post": 0.008819148875772953, "unknown": 0.011478619650006294}, "none_candidate_id": null, "predicted_none": false, "label": "courier", "correct": true, "confidence": 0.9713413715362549, "record_seconds": 0.09323890972882509, "batch_seconds": 0.09323890972882509, "batch_records": 1, "timing_scope": "record", "brier": 0.0011007582070305943, "nll": 0.029077306389808655} |
| {"record_id": "jevbench:original-adequacy-01-0", "group_id": "jevbench:original-adequacy-01", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.04194919764995575, "true": 0.9580508470535278}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9580508470535278, "record_seconds": 0.09313059784471989, "batch_seconds": 0.09313059784471989, "batch_records": 1, "timing_scope": "record", "brier": 0.0035194666124880314, "nll": 0.0428544245660305} |
| {"record_id": "jevbench:original-adequacy-01-1", "group_id": "jevbench:original-adequacy-01", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.12189864367246628, "true": 0.8781014084815979}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.8781014084815979, "record_seconds": 0.09230304509401321, "batch_seconds": 0.09230304509401321, "batch_records": 1, "timing_scope": "record", "brier": 0.029718546196818352, "nll": 0.12999318540096283} |
| {"record_id": "jevbench:original-adequacy-02-0", "group_id": "jevbench:original-adequacy-02", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.977106511592865, "true": 0.022893499583005905}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.977106511592865, "record_seconds": 0.09760473854839802, "batch_seconds": 0.09760473854839802, "batch_records": 1, "timing_scope": "record", "brier": 0.0010482240468263626, "nll": 0.023159613832831383} |
| {"record_id": "jevbench:original-adequacy-02-1", "group_id": "jevbench:original-adequacy-02", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9805435538291931, "true": 0.01945643313229084}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9805435538291931, "record_seconds": 0.09209054056555033, "batch_seconds": 0.09209054056555033, "batch_records": 1, "timing_scope": "record", "brier": 0.000757106114178896, "nll": 0.019648214802145958} |
| {"record_id": "jevbench:original-adequacy-03-0", "group_id": "jevbench:original-adequacy-03", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.8576967716217041, "true": 0.1423032283782959}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.8576967716217041, "record_seconds": 0.09129504300653934, "batch_seconds": 0.09129504300653934, "batch_records": 1, "timing_scope": "record", "brier": 0.04050041735172272, "nll": 0.1535046547651291} |
| {"record_id": "jevbench:original-adequacy-03-1", "group_id": "jevbench:original-adequacy-03", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.3137013018131256, "true": 0.686298668384552}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.686298668384552, "record_seconds": 0.09206159226596355, "batch_seconds": 0.09206159226596355, "batch_records": 1, "timing_scope": "record", "brier": 0.942011833190918, "nll": 1.1593140363693237} |
| {"record_id": "jevbench:original-adequacy-04-0", "group_id": "jevbench:original-adequacy-04", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.023224126547574997, "true": 0.9767758846282959}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9767758846282959, "record_seconds": 0.09183910675346851, "batch_seconds": 0.09183910675346851, "batch_records": 1, "timing_scope": "record", "brier": 0.0010787195060402155, "nll": 0.023498045280575752} |
| {"record_id": "jevbench:original-adequacy-04-1", "group_id": "jevbench:original-adequacy-04", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.059399258345365524, "true": 0.9406008124351501}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9406008124351501, "record_seconds": 0.0931146526709199, "batch_seconds": 0.0931146526709199, "batch_records": 1, "timing_scope": "record", "brier": 0.007056535221636295, "nll": 0.06123644486069679} |
| {"record_id": "jevbench:original-adequacy-05-0", "group_id": "jevbench:original-adequacy-05", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.5582934617996216, "true": 0.4417065680027008}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.5582934617996216, "record_seconds": 0.09237788338214159, "batch_seconds": 0.09237788338214159, "batch_records": 1, "timing_scope": "record", "brier": 0.3902093470096588, "nll": 0.5828705430030823} |
| {"record_id": "jevbench:original-adequacy-05-1", "group_id": "jevbench:original-adequacy-05", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9445129632949829, "true": 0.05548696592450142}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9445129632949829, "record_seconds": 0.09324487019330263, "batch_seconds": 0.09324487019330263, "batch_records": 1, "timing_scope": "record", "brier": 0.006157614756375551, "nll": 0.05708586797118187} |
| {"record_id": "jevbench:original-adequacy-06-0", "group_id": "jevbench:original-adequacy-06", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.01822129264473915, "true": 0.9817787408828735}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9817787408828735, "record_seconds": 0.09232437144964933, "batch_seconds": 0.09232437144964933, "batch_records": 1, "timing_scope": "record", "brier": 0.0006640297942794859, "nll": 0.01838931068778038} |
| {"record_id": "jevbench:original-adequacy-06-1", "group_id": "jevbench:original-adequacy-06", "question_id": "decision", "source": "jevbench/original/adequacy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.14822274446487427, "true": 0.8517772555351257}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.8517772555351257, "record_seconds": 0.09414408728480339, "batch_seconds": 0.09414408728480339, "batch_records": 1, "timing_scope": "record", "brier": 0.04393996298313141, "nll": 0.16043022274971008} |
| {"record_id": "jevbench:original-routing-01-0", "group_id": "jevbench:original-routing-01", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "math", "probabilities": {"coding": 0.12167800217866898, "coding_agent": 0.010152429342269897, "document": 0.017108844593167305, "general": 0.08272901922464371, "math": 0.754420280456543, "tools": 0.013911467045545578}, "none_candidate_id": null, "predicted_none": false, "label": "math", "correct": true, "confidence": 0.754420280456543, "record_seconds": 0.10141107253730297, "batch_seconds": 0.10141107253730297, "batch_records": 1, "timing_scope": "record", "brier": 0.08254833519458771, "nll": 0.2818056643009186} |
| {"record_id": "jevbench:original-routing-01-1", "group_id": "jevbench:original-routing-01", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "math", "probabilities": {"coding": 0.09723487496376038, "coding_agent": 0.010259738191962242, "document": 0.01573409140110016, "general": 0.06525596976280212, "math": 0.7983852028846741, "tools": 0.013130003586411476}, "none_candidate_id": null, "predicted_none": false, "label": "math", "correct": true, "confidence": 0.7983852028846741, "record_seconds": 0.0961416820064187, "batch_seconds": 0.0961416820064187, "batch_records": 1, "timing_scope": "record", "brier": 0.05488670989871025, "nll": 0.22516408562660217} |
| {"record_id": "jevbench:original-routing-02-0", "group_id": "jevbench:original-routing-02", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "coding", "probabilities": {"coding": 0.9079664945602417, "coding_agent": 0.03135162591934204, "document": 0.0138178002089262, "general": 0.03234141319990158, "math": 0.005961748771369457, "tools": 0.008560978807508945}, "none_candidate_id": null, "predicted_none": false, "label": "coding", "correct": true, "confidence": 0.9079664945602417, "record_seconds": 0.09519558493047953, "batch_seconds": 0.09519558493047953, "batch_records": 1, "timing_scope": "record", "brier": 0.010798821225762367, "nll": 0.09654780477285385} |
| {"record_id": "jevbench:original-routing-02-1", "group_id": "jevbench:original-routing-02", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "coding", "probabilities": {"coding": 0.8653421998023987, "coding_agent": 0.06730849295854568, "document": 0.013418884947896004, "general": 0.038165658712387085, "math": 0.0067542316392064095, "tools": 0.009010606445372105}, "none_candidate_id": null, "predicted_none": false, "label": "coding", "correct": true, "confidence": 0.8653421998023987, "record_seconds": 0.09373429697006941, "batch_seconds": 0.09373429697006941, "batch_records": 1, "timing_scope": "record", "brier": 0.024426652118563652, "nll": 0.14463023841381073} |
| {"record_id": "jevbench:original-routing-03-0", "group_id": "jevbench:original-routing-03", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "coding_agent", "probabilities": {"coding": 0.022227607667446136, "coding_agent": 0.9111711382865906, "document": 0.008895657025277615, "general": 0.036317843943834305, "math": 0.004892469383776188, "tools": 0.01649521291255951}, "none_candidate_id": null, "predicted_none": false, "label": "coding_agent", "correct": true, "confidence": 0.9111711382865906, "record_seconds": 0.0934127401560545, "batch_seconds": 0.0934127401560545, "batch_records": 1, "timing_scope": "record", "brier": 0.01007877942174673, "nll": 0.09302454441785812} |
| {"record_id": "jevbench:original-routing-03-1", "group_id": "jevbench:original-routing-03", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "coding_agent", "probabilities": {"coding": 0.022675959393382072, "coding_agent": 0.9241082072257996, "document": 0.008326338604092598, "general": 0.029346033930778503, "math": 0.003814377123489976, "tools": 0.011729126796126366}, "none_candidate_id": null, "predicted_none": false, "label": "coding_agent", "correct": true, "confidence": 0.9241082072257996, "record_seconds": 0.09385218750685453, "batch_seconds": 0.09385218750685453, "batch_records": 1, "timing_scope": "record", "brier": 0.007356403395533562, "nll": 0.07892610877752304} |
| {"record_id": "jevbench:original-routing-04-0", "group_id": "jevbench:original-routing-04", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "document", "probabilities": {"coding": 0.028274549171328545, "coding_agent": 0.029953479766845703, "document": 0.5683656930923462, "general": 0.28301742672920227, "math": 0.012231791391968727, "tools": 0.07815704494714737}, "none_candidate_id": null, "predicted_none": false, "label": "document", "correct": true, "confidence": 0.5683656930923462, "record_seconds": 0.09248057380318642, "batch_seconds": 0.09248057380318642, "batch_records": 1, "timing_scope": "record", "brier": 0.27436184883117676, "nll": 0.564990222454071} |
| {"record_id": "jevbench:original-routing-04-1", "group_id": "jevbench:original-routing-04", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "general", "probabilities": {"coding": 0.07910457998514175, "coding_agent": 0.030942222103476524, "document": 0.3704688549041748, "general": 0.44560009241104126, "math": 0.009024488739669323, "tools": 0.06485974788665771}, "none_candidate_id": null, "predicted_none": false, "label": "document", "correct": false, "confidence": 0.44560009241104126, "record_seconds": 0.09463021252304316, "batch_seconds": 0.09463021252304316, "batch_records": 1, "timing_scope": "record", "brier": 0.6063721179962158, "nll": 0.9929859042167664} |
| {"record_id": "jevbench:original-routing-05-0", "group_id": "jevbench:original-routing-05", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "tools", "probabilities": {"coding": 0.014231348410248756, "coding_agent": 0.009229802526533604, "document": 0.017179448157548904, "general": 0.17095500230789185, "math": 0.004606717731803656, "tools": 0.7837976813316345}, "none_candidate_id": null, "predicted_none": false, "label": "tools", "correct": true, "confidence": 0.7837976813316345, "record_seconds": 0.09790718089789152, "batch_seconds": 0.09790718089789152, "batch_records": 1, "timing_scope": "record", "brier": 0.07657312601804733, "nll": 0.24360434710979462} |
| {"record_id": "jevbench:original-routing-05-1", "group_id": "jevbench:original-routing-05", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "tools", "probabilities": {"coding": 0.020773598924279213, "coding_agent": 0.011663392186164856, "document": 0.02207622304558754, "general": 0.464668869972229, "math": 0.005348571110516787, "tools": 0.47546935081481934}, "none_candidate_id": null, "predicted_none": false, "label": "tools", "correct": true, "confidence": 0.47546935081481934, "record_seconds": 0.09438046347349882, "batch_seconds": 0.09438046347349882, "batch_records": 1, "timing_scope": "record", "brier": 0.49213308095932007, "nll": 0.7434528470039368} |
| {"record_id": "jevbench:original-routing-06-0", "group_id": "jevbench:original-routing-06", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "general", "probabilities": {"coding": 0.03491036966443062, "coding_agent": 0.012941834516823292, "document": 0.05313811078667641, "general": 0.8599520921707153, "math": 0.006896965205669403, "tools": 0.03216063603758812}, "none_candidate_id": null, "predicted_none": false, "label": "general", "correct": true, "confidence": 0.8599520921707153, "record_seconds": 0.09751314856112003, "batch_seconds": 0.09751314856112003, "batch_records": 1, "timing_scope": "record", "brier": 0.02490517497062683, "nll": 0.15087859332561493} |
| {"record_id": "jevbench:original-routing-06-1", "group_id": "jevbench:original-routing-06", "question_id": "decision", "source": "jevbench/original/routing", "type": "choice", "prediction": "general", "probabilities": {"coding": 0.04296352341771126, "coding_agent": 0.017565259709954262, "document": 0.0519852414727211, "general": 0.8529186248779297, "math": 0.0076102642342448235, "tools": 0.02695716731250286}, "none_candidate_id": null, "predicted_none": false, "label": "general", "correct": true, "confidence": 0.8529186248779297, "record_seconds": 0.10617559216916561, "batch_seconds": 0.10617559216916561, "batch_records": 1, "timing_scope": "record", "brier": 0.02727440372109413, "nll": 0.15909112989902496} |
| {"record_id": "jevbench:easy-intent-00", "group_id": "jevbench:easy-intent-00", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "track_order", "probabilities": {"billing_question": 0.0059721944853663445, "cancel_order": 0.009222500026226044, "change_address": 0.007862274535000324, "report_damage": 0.0065606869757175446, "track_order": 0.9703823328018188}, "none_candidate_id": null, "predicted_none": false, "label": "track_order", "correct": true, "confidence": 0.9703823328018188, "record_seconds": 0.10032523889094591, "batch_seconds": 0.10032523889094591, "batch_records": 1, "timing_scope": "record", "brier": 0.001102785812690854, "nll": 0.030065126717090607} |
| {"record_id": "jevbench:easy-intent-01", "group_id": "jevbench:easy-intent-01", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "cancel_order", "probabilities": {"billing_question": 0.0028337554540485144, "cancel_order": 0.9893297553062439, "change_address": 0.002344753360375762, "report_damage": 0.002607921604067087, "track_order": 0.0028838429134339094}, "none_candidate_id": null, "predicted_none": false, "label": "cancel_order", "correct": true, "confidence": 0.9893297553062439, "record_seconds": 0.10169948823750019, "batch_seconds": 0.10169948823750019, "batch_records": 1, "timing_scope": "record", "brier": 0.00014249997911974788, "nll": 0.010727579705417156} |
| {"record_id": "jevbench:easy-intent-02", "group_id": "jevbench:easy-intent-02", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "change_address", "probabilities": {"billing_question": 0.004088675137609243, "cancel_order": 0.0037118811160326004, "change_address": 0.9842633605003357, "report_damage": 0.002642957726493478, "track_order": 0.005293096415698528}, "none_candidate_id": null, "predicted_none": false, "label": "change_address", "correct": true, "confidence": 0.9842633605003357, "record_seconds": 0.10134199820458889, "batch_seconds": 0.10134199820458889, "batch_records": 1, "timing_scope": "record", "brier": 0.00031313925865106285, "nll": 0.01586177572607994} |
| {"record_id": "jevbench:easy-intent-03", "group_id": "jevbench:easy-intent-03", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "report_damage", "probabilities": {"billing_question": 0.0034147719852626324, "cancel_order": 0.010235723108053207, "change_address": 0.004144406411796808, "report_damage": 0.9769771099090576, "track_order": 0.00522798765450716}, "none_candidate_id": null, "predicted_none": false, "label": "report_damage", "correct": true, "confidence": 0.9769771099090576, "record_seconds": 0.09966079983860254, "batch_seconds": 0.09966079983860254, "batch_records": 1, "timing_scope": "record", "brier": 0.0006909921066835523, "nll": 0.0232920553535223} |
| {"record_id": "jevbench:easy-intent-04", "group_id": "jevbench:easy-intent-04", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "billing_question", "probabilities": {"billing_question": 0.9564553499221802, "cancel_order": 0.01075525488704443, "change_address": 0.006230953615158796, "report_damage": 0.008930988609790802, "track_order": 0.01762731932103634}, "none_candidate_id": null, "predicted_none": false, "label": "billing_question", "correct": true, "confidence": 0.9564553499221802, "record_seconds": 0.09732983075082302, "batch_seconds": 0.09732983075082302, "batch_records": 1, "timing_scope": "record", "brier": 0.0024411219637840986, "nll": 0.04452117159962654} |
| {"record_id": "jevbench:easy-intent-05", "group_id": "jevbench:easy-intent-05", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "track_order", "probabilities": {"billing_question": 0.00664381543174386, "cancel_order": 0.009760287590324879, "change_address": 0.007667530328035355, "report_damage": 0.007109439931809902, "track_order": 0.9688189625740051}, "none_candidate_id": null, "predicted_none": false, "label": "track_order", "correct": true, "confidence": 0.9688189625740051, "record_seconds": 0.09393201489001513, "batch_seconds": 0.09393201489001513, "batch_records": 1, "timing_scope": "record", "brier": 0.001220995793119073, "nll": 0.03167751431465149} |
| {"record_id": "jevbench:easy-intent-06", "group_id": "jevbench:easy-intent-06", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "set_alarm", "probabilities": {"play_music": 0.00409364327788353, "send_message": 0.005844433326274157, "set_alarm": 0.982750415802002, "turn_off_lights": 0.00420930003747344, "weather": 0.0031022285111248493}, "none_candidate_id": null, "predicted_none": false, "label": "set_alarm", "correct": true, "confidence": 0.982750415802002, "record_seconds": 0.09427301026880741, "batch_seconds": 0.09427301026880741, "batch_records": 1, "timing_scope": "record", "brier": 0.00037580550997518003, "nll": 0.017400091513991356} |
| {"record_id": "jevbench:easy-intent-07", "group_id": "jevbench:easy-intent-07", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "play_music", "probabilities": {"play_music": 0.984415590763092, "send_message": 0.004624376073479652, "set_alarm": 0.0040607936680316925, "turn_off_lights": 0.0036733215674757957, "weather": 0.0032258753199130297}, "none_candidate_id": null, "predicted_none": false, "label": "play_music", "correct": true, "confidence": 0.984415590763092, "record_seconds": 0.09519085939973593, "batch_seconds": 0.09519085939973593, "batch_records": 1, "timing_scope": "record", "brier": 0.0003046482743229717, "nll": 0.015707122161984444} |
| {"record_id": "jevbench:easy-intent-08", "group_id": "jevbench:easy-intent-08", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "weather", "probabilities": {"play_music": 0.003520966973155737, "send_message": 0.004435673356056213, "set_alarm": 0.004580864682793617, "turn_off_lights": 0.00375518761575222, "weather": 0.9837072491645813}, "none_candidate_id": null, "predicted_none": false, "label": "weather", "correct": true, "confidence": 0.9837072491645813, "record_seconds": 0.09346645977348089, "batch_seconds": 0.09346645977348089, "batch_records": 1, "timing_scope": "record", "brier": 0.0003326119331177324, "nll": 0.01642693765461445} |
| {"record_id": "jevbench:easy-intent-09", "group_id": "jevbench:easy-intent-09", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "send_message", "probabilities": {"play_music": 0.005952137056738138, "send_message": 0.9733918309211731, "set_alarm": 0.009192529134452343, "turn_off_lights": 0.00546400947496295, "weather": 0.005999506451189518}, "none_candidate_id": null, "predicted_none": false, "label": "send_message", "correct": true, "confidence": 0.9733918309211731, "record_seconds": 0.09259012155234814, "batch_seconds": 0.09259012155234814, "batch_records": 1, "timing_scope": "record", "brier": 0.0008937746752053499, "nll": 0.02696857415139675} |
| {"record_id": "jevbench:easy-intent-10", "group_id": "jevbench:easy-intent-10", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "turn_off_lights", "probabilities": {"play_music": 0.005629140883684158, "send_message": 0.006895106285810471, "set_alarm": 0.009182171896100044, "turn_off_lights": 0.9734789133071899, "weather": 0.004814653191715479}, "none_candidate_id": null, "predicted_none": false, "label": "turn_off_lights", "correct": true, "confidence": 0.9734789133071899, "record_seconds": 0.09222028590738773, "batch_seconds": 0.09222028590738773, "batch_records": 1, "timing_scope": "record", "brier": 0.0008900909451767802, "nll": 0.026879115030169487} |
| {"record_id": "jevbench:easy-intent-11", "group_id": "jevbench:easy-intent-11", "question_id": "decision", "source": "jevbench/easy/intent", "type": "choice", "prediction": "set_alarm", "probabilities": {"play_music": 0.0024051768705248833, "send_message": 0.0018757006619125605, "set_alarm": 0.9921075701713562, "turn_off_lights": 0.001957740169018507, "weather": 0.0016537545016035438}, "none_candidate_id": null, "predicted_none": false, "label": "set_alarm", "correct": true, "confidence": 0.9921075701713562, "record_seconds": 0.09173869714140892, "batch_seconds": 0.09173869714140892, "batch_records": 1, "timing_scope": "record", "brier": 7.81612325226888e-05, "nll": 0.007923739962279797} |
| {"record_id": "jevbench:easy-fact-00", "group_id": "jevbench:easy-fact-00", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.00628479290753603, "true": 0.993715226650238}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.993715226650238, "record_seconds": 0.09195497911423445, "batch_seconds": 0.09195497911423445, "batch_records": 1, "timing_scope": "record", "brier": 7.899699994595721e-05, "nll": 0.0063046058639883995} |
| {"record_id": "jevbench:easy-fact-01", "group_id": "jevbench:easy-fact-01", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9877414703369141, "true": 0.012258533388376236}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9877414703369141, "record_seconds": 0.09179344680160284, "batch_seconds": 0.09179344680160284, "batch_records": 1, "timing_scope": "record", "brier": 0.00030054320814087987, "nll": 0.01233428530395031} |
| {"record_id": "jevbench:easy-fact-02", "group_id": "jevbench:easy-fact-02", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.005585193634033203, "true": 0.9944148063659668}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9944148063659668, "record_seconds": 0.09218625351786613, "batch_seconds": 0.09218625351786613, "batch_records": 1, "timing_scope": "record", "brier": 6.238877540454268e-05, "nll": 0.005600849166512489} |
| {"record_id": "jevbench:easy-fact-03", "group_id": "jevbench:easy-fact-03", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9873358607292175, "true": 0.012664132751524448}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9873358607292175, "record_seconds": 0.09130692388862371, "batch_seconds": 0.09130692388862371, "batch_records": 1, "timing_scope": "record", "brier": 0.0003207606787327677, "nll": 0.01274501346051693} |
| {"record_id": "jevbench:easy-fact-04", "group_id": "jevbench:easy-fact-04", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.005595398135483265, "true": 0.9944045543670654}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9944045543670654, "record_seconds": 0.09932985436171293, "batch_seconds": 0.09932985436171293, "batch_records": 1, "timing_scope": "record", "brier": 6.261749513214454e-05, "nll": 0.005611158907413483} |
| {"record_id": "jevbench:easy-fact-05", "group_id": "jevbench:easy-fact-05", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9879918694496155, "true": 0.012008130550384521}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9879918694496155, "record_seconds": 0.10643925052136183, "batch_seconds": 0.10643925052136183, "batch_records": 1, "timing_scope": "record", "brier": 0.00028839040896855295, "nll": 0.012080810964107513} |
| {"record_id": "jevbench:easy-fact-06", "group_id": "jevbench:easy-fact-06", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.008821193128824234, "true": 0.9911788105964661}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9911788105964661, "record_seconds": 0.11247524991631508, "batch_seconds": 0.11247524991631508, "batch_records": 1, "timing_scope": "record", "brier": 0.00015562682528980076, "nll": 0.008860326372087002} |
| {"record_id": "jevbench:easy-fact-07", "group_id": "jevbench:easy-fact-07", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.988893449306488, "true": 0.011106510646641254}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.988893449306488, "record_seconds": 0.11081347148865461, "batch_seconds": 0.11081347148865461, "batch_records": 1, "timing_scope": "record", "brier": 0.0002467100275680423, "nll": 0.011168688535690308} |
| {"record_id": "jevbench:easy-fact-08", "group_id": "jevbench:easy-fact-08", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.004620465449988842, "true": 0.9953795671463013}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9953795671463013, "record_seconds": 0.1001509502530098, "batch_seconds": 0.1001509502530098, "batch_records": 1, "timing_scope": "record", "brier": 4.269710188964382e-05, "nll": 0.004631140269339085} |
| {"record_id": "jevbench:easy-fact-09", "group_id": "jevbench:easy-fact-09", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9881842136383057, "true": 0.01181579940021038}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9881842136383057, "record_seconds": 0.09356402140110731, "batch_seconds": 0.09356402140110731, "batch_records": 1, "timing_scope": "record", "brier": 0.00027922593289986253, "nll": 0.011886147782206535} |
| {"record_id": "jevbench:easy-fact-10", "group_id": "jevbench:easy-fact-10", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "true", "probabilities": {"false": 0.025065023452043533, "true": 0.974934995174408}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.974934995174408, "record_seconds": 0.0928747234866023, "batch_seconds": 0.0928747234866023, "batch_records": 1, "timing_scope": "record", "brier": 0.001256509916856885, "nll": 0.025384481996297836} |
| {"record_id": "jevbench:easy-fact-11", "group_id": "jevbench:easy-fact-11", "question_id": "decision", "source": "jevbench/easy/fact", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9848653078079224, "true": 0.015134665183722973}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9848653078079224, "record_seconds": 0.09189386665821075, "batch_seconds": 0.09189386665821075, "batch_records": 1, "timing_scope": "record", "brier": 0.00045811699237674475, "nll": 0.015250390395522118} |
| {"record_id": "jevbench:easy-extraction-00", "group_id": "jevbench:easy-extraction-00", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "paypal", "probabilities": {"bank_transfer": 0.0023095733486115932, "cash": 0.0016162109095603228, "credit_card": 0.0031068960670381784, "paypal": 0.9929673671722412}, "none_candidate_id": null, "predicted_none": false, "label": "paypal", "correct": true, "confidence": 0.9929673671722412, "record_seconds": 0.09179357811808586, "batch_seconds": 0.09179357811808586, "batch_records": 1, "timing_scope": "record", "brier": 6.70569934300147e-05, "nll": 0.0070574781857430935} |
| {"record_id": "jevbench:easy-extraction-01", "group_id": "jevbench:easy-extraction-01", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "cash", "probabilities": {"bank_transfer": 0.0033479572739452124, "cash": 0.9899790287017822, "credit_card": 0.0035644066520035267, "paypal": 0.003108628559857607}, "none_candidate_id": null, "predicted_none": false, "label": "cash", "correct": true, "confidence": 0.9899790287017822, "record_seconds": 0.09130608662962914, "batch_seconds": 0.09130608662962914, "batch_records": 1, "timing_scope": "record", "brier": 0.00013399725139606744, "nll": 0.010071518830955029} |
| {"record_id": "jevbench:easy-extraction-02", "group_id": "jevbench:easy-extraction-02", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "bank_transfer", "probabilities": {"bank_transfer": 0.9926961660385132, "cash": 0.0018654235173016787, "credit_card": 0.0026870653964579105, "paypal": 0.002751322463154793}, "none_candidate_id": null, "predicted_none": false, "label": "bank_transfer", "correct": true, "confidence": 0.9926961660385132, "record_seconds": 0.0920643387362361, "batch_seconds": 0.0920643387362361, "batch_records": 1, "timing_scope": "record", "brier": 7.161589019233361e-05, "nll": 0.00733063742518425} |
| {"record_id": "jevbench:easy-extraction-03", "group_id": "jevbench:easy-extraction-03", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "high", "probabilities": {"high": 0.9728916883468628, "low": 0.0028448007069528103, "medium": 0.004258858971297741, "urgent": 0.020004691556096077}, "none_candidate_id": null, "predicted_none": false, "label": "high", "correct": true, "confidence": 0.9728916883468628, "record_seconds": 0.0921715646982193, "batch_seconds": 0.0921715646982193, "batch_records": 1, "timing_scope": "record", "brier": 0.0011612789239734411, "nll": 0.027482520788908005} |
| {"record_id": "jevbench:easy-extraction-04", "group_id": "jevbench:easy-extraction-04", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "low", "probabilities": {"high": 0.0032316737342625856, "low": 0.989163875579834, "medium": 0.004714059643447399, "urgent": 0.0028904264327138662}, "none_candidate_id": null, "predicted_none": false, "label": "low", "correct": true, "confidence": 0.989163875579834, "record_seconds": 0.09155805129557848, "batch_seconds": 0.09155805129557848, "batch_records": 1, "timing_scope": "record", "brier": 0.00015844222798477858, "nll": 0.010895262472331524} |
| {"record_id": "jevbench:easy-extraction-05", "group_id": "jevbench:easy-extraction-05", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "urgent", "probabilities": {"high": 0.02061060257256031, "low": 0.003915841691195965, "medium": 0.0058283028192818165, "urgent": 0.9696452021598816}, "none_candidate_id": null, "predicted_none": false, "label": "urgent", "correct": true, "confidence": 0.9696452021598816, "record_seconds": 0.09136008378118277, "batch_seconds": 0.09136008378118277, "batch_records": 1, "timing_scope": "record", "brier": 0.0013955136528238654, "nll": 0.030825044959783554} |
| {"record_id": "jevbench:easy-extraction-06", "group_id": "jevbench:easy-extraction-06", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "M", "probabilities": {"L": 0.004450364038348198, "M": 0.9880393147468567, "S": 0.004133453592658043, "XL": 0.003376852022483945}, "none_candidate_id": null, "predicted_none": false, "label": "M", "correct": true, "confidence": 0.9880393147468567, "record_seconds": 0.09124485310167074, "batch_seconds": 0.09124485310167074, "batch_records": 1, "timing_scope": "record", "brier": 0.00019135228649247438, "nll": 0.01203279010951519} |
| {"record_id": "jevbench:easy-extraction-07", "group_id": "jevbench:easy-extraction-07", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "XL", "probabilities": {"L": 0.030208894982933998, "M": 0.010774485766887665, "S": 0.008307535201311111, "XL": 0.9507090449333191}, "none_candidate_id": null, "predicted_none": false, "label": "XL", "correct": true, "confidence": 0.9507090449333191, "record_seconds": 0.09175907541066408, "batch_seconds": 0.09175907541066408, "batch_records": 1, "timing_scope": "record", "brier": 0.0035272804088890553, "nll": 0.05054720863699913} |
| {"record_id": "jevbench:easy-extraction-08", "group_id": "jevbench:easy-extraction-08", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "thursday", "probabilities": {"friday": 0.002606073161587119, "monday": 0.0018341278191655874, "thursday": 0.9898824095726013, "tuesday": 0.002358001656830311, "wednesday": 0.003319323994219303}, "none_candidate_id": null, "predicted_none": false, "label": "thursday", "correct": true, "confidence": 0.9898824095726013, "record_seconds": 0.09037533216178417, "batch_seconds": 0.09037533216178417, "batch_records": 1, "timing_scope": "record", "brier": 0.0001290993532165885, "nll": 0.01016912143677473} |
| {"record_id": "jevbench:easy-extraction-09", "group_id": "jevbench:easy-extraction-09", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "monday", "probabilities": {"friday": 0.015424872748553753, "monday": 0.9404411315917969, "thursday": 0.010920117609202862, "tuesday": 0.02110033854842186, "wednesday": 0.012113548815250397}, "none_candidate_id": null, "predicted_none": false, "label": "monday", "correct": true, "confidence": 0.9404411315917969, "record_seconds": 0.09008316975086927, "batch_seconds": 0.09008316975086927, "batch_records": 1, "timing_scope": "record", "brier": 0.0044963969849050045, "nll": 0.061406224966049194} |
| {"record_id": "jevbench:easy-extraction-10", "group_id": "jevbench:easy-extraction-10", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "friday", "probabilities": {"friday": 0.9792814254760742, "monday": 0.004146328661590815, "thursday": 0.006662145722657442, "tuesday": 0.004510460887104273, "wednesday": 0.005399507004767656}, "none_candidate_id": null, "predicted_none": false, "label": "friday", "correct": true, "confidence": 0.9792814254760742, "record_seconds": 0.08984820172190666, "batch_seconds": 0.08984820172190666, "batch_records": 1, "timing_scope": "record", "brier": 0.0005403344985097647, "nll": 0.020936215296387672} |
| {"record_id": "jevbench:easy-extraction-11", "group_id": "jevbench:easy-extraction-11", "question_id": "decision", "source": "jevbench/easy/extraction", "type": "choice", "prediction": "credit_card", "probabilities": {"bank_transfer": 0.00494010467082262, "cash": 0.003895148169249296, "credit_card": 0.9851174354553223, "paypal": 0.006047319155186415}, "none_candidate_id": null, "predicted_none": false, "label": "credit_card", "correct": true, "confidence": 0.9851174354553223, "record_seconds": 0.09116359893232584, "batch_seconds": 0.09116359893232584, "batch_records": 1, "timing_scope": "record", "brier": 0.0002976375981234014, "nll": 0.014994421042501926} |
| {"record_id": "jevbench:easy-tool_selection-00", "group_id": "jevbench:easy-tool_selection-00", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "get_weather", "probabilities": {"convert_currency": 0.0022996694315224886, "create_calendar_event": 0.0028507725801318884, "get_weather": 0.9884446859359741, "send_email": 0.002876254031434655, "translate_text": 0.0035286624915897846}, "none_candidate_id": null, "predicted_none": false, "label": "get_weather", "correct": true, "confidence": 0.9884446859359741, "record_seconds": 0.09187935572117567, "batch_seconds": 0.09187935572117567, "batch_records": 1, "timing_scope": "record", "brier": 0.00016766496992204338, "nll": 0.011622595600783825} |
| {"record_id": "jevbench:easy-tool_selection-01", "group_id": "jevbench:easy-tool_selection-01", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "send_email", "probabilities": {"convert_currency": 0.0020973840728402138, "create_calendar_event": 0.0029527912847697735, "get_weather": 0.0021499597933143377, "send_email": 0.9899299740791321, "translate_text": 0.0028699298854917288}, "none_candidate_id": null, "predicted_none": false, "label": "send_email", "correct": true, "confidence": 0.9899299740791321, "record_seconds": 0.091829520650208, "batch_seconds": 0.091829520650208, "batch_records": 1, "timing_scope": "record", "brier": 0.00012738224177155644, "nll": 0.010121071711182594} |
| {"record_id": "jevbench:easy-tool_selection-02", "group_id": "jevbench:easy-tool_selection-02", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "create_calendar_event", "probabilities": {"convert_currency": 0.0018824400613084435, "create_calendar_event": 0.9896768927574158, "get_weather": 0.0021581437904387712, "send_email": 0.003920454531908035, "translate_text": 0.0023620999418199062}, "none_candidate_id": null, "predicted_none": false, "label": "create_calendar_event", "correct": true, "confidence": 0.9896768927574158, "record_seconds": 0.09144001547247171, "batch_seconds": 0.09144001547247171, "batch_records": 1, "timing_scope": "record", "brier": 0.0001357171859126538, "nll": 0.010376759804785252} |
| {"record_id": "jevbench:easy-tool_selection-03", "group_id": "jevbench:easy-tool_selection-03", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "convert_currency", "probabilities": {"convert_currency": 0.993591845035553, "create_calendar_event": 0.0012191603891551495, "get_weather": 0.0011094675865024328, "send_email": 0.0016002568881958723, "translate_text": 0.0024791788309812546}, "none_candidate_id": null, "predicted_none": false, "label": "convert_currency", "correct": true, "confidence": 0.993591845035553, "record_seconds": 0.09548612777143717, "batch_seconds": 0.09548612777143717, "batch_records": 1, "timing_scope": "record", "brier": 5.248886736808345e-05, "nll": 0.006428775377571583} |
| {"record_id": "jevbench:easy-tool_selection-04", "group_id": "jevbench:easy-tool_selection-04", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "translate_text", "probabilities": {"convert_currency": 0.0030292554292827845, "create_calendar_event": 0.0038562826812267303, "get_weather": 0.0032783763017505407, "send_email": 0.00348639371804893, "translate_text": 0.9863496422767639}, "none_candidate_id": null, "predicted_none": false, "label": "translate_text", "correct": true, "confidence": 0.9863496422767639, "record_seconds": 0.0921014342457056, "batch_seconds": 0.0921014342457056, "batch_records": 1, "timing_scope": "record", "brier": 0.00023328227689489722, "nll": 0.013744380325078964} |
| {"record_id": "jevbench:easy-tool_selection-05", "group_id": "jevbench:easy-tool_selection-05", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "create_calendar_event", "probabilities": {"convert_currency": 0.002607933711260557, "create_calendar_event": 0.9830988049507141, "get_weather": 0.002926304005086422, "send_email": 0.007982926443219185, "translate_text": 0.0033840832766145468}, "none_candidate_id": null, "predicted_none": false, "label": "create_calendar_event", "correct": true, "confidence": 0.9830988049507141, "record_seconds": 0.09178250469267368, "batch_seconds": 0.09178250469267368, "batch_records": 1, "timing_scope": "record", "brier": 0.00037619410431943834, "nll": 0.017045650631189346} |
| {"record_id": "jevbench:easy-tool_selection-06", "group_id": "jevbench:easy-tool_selection-06", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "search_flights", "probabilities": {"book_restaurant": 0.0023498237133026123, "call_taxi": 0.002656635595485568, "get_stock_price": 0.0023494907654821873, "search_flights": 0.9893094301223755, "set_reminder": 0.003334645414724946}, "none_candidate_id": null, "predicted_none": false, "label": "search_flights", "correct": true, "confidence": 0.9893094301223755, "record_seconds": 0.09201027546077967, "batch_seconds": 0.09201027546077967, "batch_records": 1, "timing_scope": "record", "brier": 0.00014350762648973614, "nll": 0.010748124681413174} |
| {"record_id": "jevbench:easy-tool_selection-07", "group_id": "jevbench:easy-tool_selection-07", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "book_restaurant", "probabilities": {"book_restaurant": 0.9872872829437256, "call_taxi": 0.0039504822343587875, "get_stock_price": 0.002249746350571513, "search_flights": 0.0022440056782215834, "set_reminder": 0.0042684683576226234}, "none_candidate_id": null, "predicted_none": false, "label": "book_restaurant", "correct": true, "confidence": 0.9872872829437256, "record_seconds": 0.091679478995502, "batch_seconds": 0.091679478995502, "batch_records": 1, "timing_scope": "record", "brier": 0.00020553624199237674, "nll": 0.012794215232133865} |
| {"record_id": "jevbench:easy-tool_selection-08", "group_id": "jevbench:easy-tool_selection-08", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "set_reminder", "probabilities": {"book_restaurant": 0.001646757242269814, "call_taxi": 0.002147497609257698, "get_stock_price": 0.0015086604980751872, "search_flights": 0.0016004343051463366, "set_reminder": 0.993096649646759}, "none_candidate_id": null, "predicted_none": false, "label": "set_reminder", "correct": true, "confidence": 0.993096649646759, "record_seconds": 0.09185116644948721, "batch_seconds": 0.09185116644948721, "batch_records": 1, "timing_scope": "record", "brier": 5.9817248256877065e-05, "nll": 0.006927288603037596} |
| {"record_id": "jevbench:easy-tool_selection-09", "group_id": "jevbench:easy-tool_selection-09", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "get_stock_price", "probabilities": {"book_restaurant": 0.0020712038967758417, "call_taxi": 0.0025696263182908297, "get_stock_price": 0.9900886416435242, "search_flights": 0.002244378440082073, "set_reminder": 0.0030261953361332417}, "none_candidate_id": null, "predicted_none": false, "label": "get_stock_price", "correct": true, "confidence": 0.9900886416435242, "record_seconds": 0.09258208144456148, "batch_seconds": 0.09258208144456148, "batch_records": 1, "timing_scope": "record", "brier": 0.0001233229850186035, "nll": 0.00996080320328474} |
| {"record_id": "jevbench:easy-tool_selection-10", "group_id": "jevbench:easy-tool_selection-10", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "call_taxi", "probabilities": {"book_restaurant": 0.0018294370966032147, "call_taxi": 0.9915422201156616, "get_stock_price": 0.001595211448147893, "search_flights": 0.0024899733252823353, "set_reminder": 0.0025431260000914335}, "none_candidate_id": null, "predicted_none": false, "label": "call_taxi", "correct": true, "confidence": 0.9915422201156616, "record_seconds": 0.093500598333776, "batch_seconds": 0.093500598333776, "batch_records": 1, "timing_scope": "record", "brier": 9.009303903440014e-05, "nll": 0.008493749424815178} |
| {"record_id": "jevbench:easy-tool_selection-11", "group_id": "jevbench:easy-tool_selection-11", "question_id": "decision", "source": "jevbench/easy/tool_selection", "type": "choice", "prediction": "set_reminder", "probabilities": {"book_restaurant": 0.0014658191939815879, "call_taxi": 0.0019141583470627666, "get_stock_price": 0.001357055502012372, "search_flights": 0.0013915670569986105, "set_reminder": 0.9938714504241943}, "none_candidate_id": null, "predicted_none": false, "label": "set_reminder", "correct": true, "confidence": 0.9938714504241943, "record_seconds": 0.092924858443439, "batch_seconds": 0.092924858443439, "batch_records": 1, "timing_scope": "record", "brier": 4.714980605058372e-05, "nll": 0.006147406063973904} |
| {"record_id": "jevbench:hard-opus-a-long_policy-01", "group_id": "jevbench:hard-opus-a-long_policy-01", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "pay_subject_to_15000_sublimit", "probabilities": {"deny_repeated_seepage": 0.043275136500597, "deny_vacancy_exclusion": 0.031198417767882347, "pay_full_estimate_less_deductible": 0.050891418009996414, "pay_subject_to_10000_sublimit": 0.018289444968104362, "pay_subject_to_15000_sublimit": 0.8563455939292908}, "none_candidate_id": null, "predicted_none": false, "label": "pay_subject_to_15000_sublimit", "correct": true, "confidence": 0.8563455939292908, "record_seconds": 3.522061969153583, "batch_seconds": 3.522061969153583, "batch_records": 1, "timing_scope": "record", "brier": 0.02640710584819317, "nll": 0.15508125722408295} |
| {"record_id": "jevbench:hard-opus-a-long_policy-04", "group_id": "jevbench:hard-opus-a-long_policy-04", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "vp_and_finance_director", "probabilities": {"budget_holder": 0.03375067189335823, "cfo": 0.058348994702100754, "department_head": 0.18632780015468597, "executive_committee": 0.01337407436221838, "vp_and_finance_director": 0.7081984281539917}, "none_candidate_id": null, "predicted_none": false, "label": "cfo", "correct": false, "confidence": 0.7081984281539917, "record_seconds": 2.3210465209558606, "batch_seconds": 2.3210465209558606, "batch_records": 1, "timing_scope": "record", "brier": 1.4242875576019287, "nll": 2.841313123703003} |
| {"record_id": "jevbench:hard-opus-a-long_policy-08", "group_id": "jevbench:hard-opus-a-long_policy-08", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "parts_covered_labour_charged", "probabilities": {"full_repair_no_charge": 0.2269529104232788, "not_covered": 0.1140434667468071, "parts_covered_labour_charged": 0.5996620655059814, "pro_rata_replacement_credit": 0.05934153124690056}, "none_candidate_id": null, "predicted_none": false, "label": "parts_covered_labour_charged", "correct": true, "confidence": 0.5996620655059814, "record_seconds": 1.7285670787096024, "batch_seconds": 1.7285670787096024, "batch_records": 1, "timing_scope": "record", "brier": 0.2283054143190384, "nll": 0.5113890171051025} |
| {"record_id": "jevbench:hard-opus-a-long_policy-09", "group_id": "jevbench:hard-opus-a-long_policy-09", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "license_required", "probabilities": {"license_exception_lvs": 0.0596235953271389, "license_required": 0.8769132494926453, "no_license_required": 0.05002553388476372, "shipment_prohibited": 0.013437572866678238}, "none_candidate_id": null, "predicted_none": false, "label": "license_required", "correct": true, "confidence": 0.8769132494926453, "record_seconds": 1.819549304433167, "batch_seconds": 1.819549304433167, "batch_records": 1, "timing_scope": "record", "brier": 0.021388445049524307, "nll": 0.13134720921516418} |
| {"record_id": "jevbench:hard-opus-a-long_policy-11", "group_id": "jevbench:hard-opus-a-long_policy-11", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.7772389054298401, "true": 0.2227611094713211}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": false, "confidence": 0.7772389054298401, "record_seconds": 0.8428583880886436, "batch_seconds": 0.8428583880886436, "batch_records": 1, "timing_scope": "record", "brier": 1.2082005739212036, "nll": 1.5016553401947021} |
| {"record_id": "jevbench:hard-opus-a-long_policy-13", "group_id": "jevbench:hard-opus-a-long_policy-13", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "false", "probabilities": {"false": 0.8114722371101379, "true": 0.18852773308753967}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": false, "confidence": 0.8114722371101379, "record_seconds": 0.9154827119782567, "batch_seconds": 0.9154827119782567, "batch_records": 1, "timing_scope": "record", "brier": 1.316974401473999, "nll": 1.6685101985931396} |
| {"record_id": "jevbench:hard-opus-a-long_policy-17", "group_id": "jevbench:hard-opus-a-long_policy-17", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "database_oncall", "probabilities": {"database_oncall": 0.4228275716304779, "no_page_ticket_only": 0.18430973589420319, "platform_oncall": 0.21236754953861237, "security_incident_response": 0.10227508097887039, "vendor_support": 0.07822002470493317}, "none_candidate_id": null, "predicted_none": false, "label": "database_oncall", "correct": true, "confidence": 0.4228275716304779, "record_seconds": 2.1439826702699065, "batch_seconds": 2.1439826702699065, "batch_records": 1, "timing_scope": "record", "brier": 0.4287765920162201, "nll": 0.8607907891273499} |
| {"record_id": "jevbench:hard-opus-a-long_policy-19", "group_id": "jevbench:hard-opus-a-long_policy-19", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.26175037026405334, "true": 0.738249659538269}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.738249659538269, "record_seconds": 0.8770911106839776, "batch_seconds": 0.8770911106839776, "batch_records": 1, "timing_scope": "record", "brier": 1.0900250673294067, "nll": 1.3403639793395996} |
| {"record_id": "jevbench:hard-opus-a-probability-03", "group_id": "jevbench:hard-opus-a-probability-03", "question_id": "decision", "source": "jevbench/hard/probability", "type": "noul", "prediction": "true", "probabilities": {"false": 0.1886514127254486, "true": 0.8113486170768738}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.8113486170768738, "record_seconds": 0.1707887863740325, "batch_seconds": 0.1707887863740325, "batch_records": 1, "timing_scope": "record", "brier": 0.07461276650428772, "nll": 0.7660260796546936} |
| {"record_id": "jevbench:hard-opus-a-probability-04", "group_id": "jevbench:hard-opus-a-probability-04", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "late", "probabilities": {"early": 0.08892896771430969, "late": 0.5722838640213013, "on_time": 0.33878710865974426}, "none_candidate_id": null, "predicted_none": false, "label": "on_time", "correct": false, "confidence": 0.5722838640213013, "record_seconds": 0.5928696068003774, "batch_seconds": 0.5928696068003774, "batch_records": 1, "timing_scope": "record", "brier": 0.12583960592746735, "nll": 0.9968599677085876} |
| {"record_id": "jevbench:hard-opus-a-probability-07", "group_id": "jevbench:hard-opus-a-probability-07", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "resolved_first_contact", "probabilities": {"escalated": 0.2519245743751526, "reopened": 0.035984333604574203, "resolved_first_contact": 0.7120911478996277}, "none_candidate_id": null, "predicted_none": false, "label": "escalated", "correct": false, "confidence": 0.7120911478996277, "record_seconds": 0.6649914914742112, "batch_seconds": 0.6649914914742112, "batch_records": 1, "timing_scope": "record", "brier": 0.38780564069747925, "nll": 1.2809621095657349} |
| {"record_id": "jevbench:hard-opus-a-probability-08", "group_id": "jevbench:hard-opus-a-probability-08", "question_id": "decision", "source": "jevbench/hard/probability", "type": "noul", "prediction": "false", "probabilities": {"false": 0.5380406379699707, "true": 0.4619593322277069}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.5380406379699707, "record_seconds": 0.1576939644291997, "batch_seconds": 0.1576939644291997, "batch_records": 1, "timing_scope": "record", "brier": 0.059140030294656754, "nll": 0.6640337705612183} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-03", "group_id": "jevbench:hard-opus-a-temporal_numeric-03", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "prediction": "true", "probabilities": {"false": 0.4699100852012634, "true": 0.5300899147987366}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.5300899147987366, "record_seconds": 0.19449108000844717, "batch_seconds": 0.19449108000844717, "batch_records": 1, "timing_scope": "record", "brier": 0.4416309893131256, "nll": 0.6347086429595947} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-06", "group_id": "jevbench:hard-opus-a-temporal_numeric-06", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "prediction": "false", "probabilities": {"false": 0.7531299591064453, "true": 0.2468699961900711}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.7531299591064453, "record_seconds": 0.19994631223380566, "batch_seconds": 0.19994631223380566, "batch_records": 1, "timing_scope": "record", "brier": 0.1218896135687828, "nll": 0.2835174798965454} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-07", "group_id": "jevbench:hard-opus-a-temporal_numeric-07", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "sep_27", "probabilities": {"sep_24": 0.17612063884735107, "sep_25": 0.18287689983844757, "sep_26": 0.19484296441078186, "sep_27": 0.26433631777763367, "sep_28": 0.18182313442230225}, "none_candidate_id": null, "predicted_none": false, "label": "sep_26", "correct": false, "confidence": 0.26433631777763367, "record_seconds": 1.0923941880464554, "batch_seconds": 1.0923941880464554, "batch_records": 1, "timing_scope": "record", "brier": 0.8156736493110657, "nll": 1.6355613470077515} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-09", "group_id": "jevbench:hard-opus-a-temporal_numeric-09", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "prediction": "true", "probabilities": {"false": 0.45387688279151917, "true": 0.5461231470108032}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.5461231470108032, "record_seconds": 0.19938875176012516, "batch_seconds": 0.19938875176012516, "batch_records": 1, "timing_scope": "record", "brier": 0.5965009927749634, "nll": 0.7899293303489685} |
| {"record_id": "jevbench:hard-opus-a-temporal_numeric-12", "group_id": "jevbench:hard-opus-a-temporal_numeric-12", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "score", "prediction": "2", "probabilities": {"0": 0.08988243341445923, "1": 0.21390636265277863, "2": 0.3656686842441559, "3": 0.33054256439208984}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": false, "confidence": 0.3656686842441559, "record_seconds": 0.4228637758642435, "batch_seconds": 0.4228637758642435, "batch_records": 1, "timing_scope": "record", "brier": 0.8689941167831421, "nll": 1.5422168970108032} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-02", "group_id": "jevbench:hard-opus-b-ambiguous-02", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "real_incident", "probabilities": {"cannot_determine": 0.10829369723796844, "false_positive": 0.09716939181089401, "real_incident": 0.794536828994751}, "none_candidate_id": null, "predicted_none": false, "label": "real_incident", "correct": true, "confidence": 0.794536828994751, "record_seconds": 0.2930323285982013, "batch_seconds": 0.2930323285982013, "batch_records": 1, "timing_scope": "record", "brier": 0.0633845329284668, "nll": 0.2299959361553192} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-03", "group_id": "jevbench:hard-opus-b-ambiguous-03", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "within_limitation", "probabilities": {"insufficient_information": 0.20377543568611145, "time_barred": 0.35523882508277893, "within_limitation": 0.4409857392311096}, "none_candidate_id": null, "predicted_none": false, "label": "time_barred", "correct": false, "confidence": 0.4409857392311096, "record_seconds": 0.25823462568223476, "batch_seconds": 0.25823462568223476, "batch_records": 1, "timing_scope": "record", "brier": 0.6517098546028137, "nll": 1.034964919090271} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-07", "group_id": "jevbench:hard-opus-b-ambiguous-07", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "over_limit", "probabilities": {"insufficient_information": 0.03438777104020119, "over_limit": 0.7292750477790833, "within_policy": 0.23633717000484467}, "none_candidate_id": null, "predicted_none": false, "label": "within_policy", "correct": false, "confidence": 0.7292750477790833, "record_seconds": 0.2709821034222841, "batch_seconds": 0.2709821034222841, "batch_records": 1, "timing_scope": "record", "brier": 1.1162054538726807, "nll": 1.4424958229064941} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-09", "group_id": "jevbench:hard-opus-b-ambiguous-09", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "compromised", "probabilities": {"compromised": 0.4495149552822113, "inconclusive": 0.10455483198165894, "legitimate": 0.4459301829338074}, "none_candidate_id": null, "predicted_none": false, "label": "legitimate", "correct": false, "confidence": 0.4495149552822113, "record_seconds": 0.3368332237005234, "batch_seconds": 0.3368332237005234, "batch_records": 1, "timing_scope": "record", "brier": 0.5199887752532959, "nll": 0.8075928688049316} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-10", "group_id": "jevbench:hard-opus-b-ambiguous-10", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "ask_user_which_alex", "probabilities": {"ask_user_which_alex": 0.6975488662719727, "send_to_alex_moreno": 0.23038768768310547, "send_to_alexandra_lind": 0.07206343859434128}, "none_candidate_id": null, "predicted_none": false, "label": "ask_user_which_alex", "correct": true, "confidence": 0.6975488662719727, "record_seconds": 0.20556325651705265, "batch_seconds": 0.20556325651705265, "batch_records": 1, "timing_scope": "record", "brier": 0.1497483253479004, "nll": 0.3601827025413513} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-11", "group_id": "jevbench:hard-opus-b-ambiguous-11", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "cannot_determine", "probabilities": {"cannot_determine": 0.6572123765945435, "grade_correct": 0.2825537323951721, "grade_should_be_higher": 0.06023387983441353}, "none_candidate_id": null, "predicted_none": false, "label": "cannot_determine", "correct": true, "confidence": 0.6572123765945435, "record_seconds": 0.2500260332599282, "batch_seconds": 0.2500260332599282, "batch_records": 1, "timing_scope": "record", "brier": 0.20096808671951294, "nll": 0.41974806785583496} |
| {"record_id": "jevbench:hard-opus-b-ambiguous-13", "group_id": "jevbench:hard-opus-b-ambiguous-13", "question_id": "decision", "source": "jevbench/hard/ambiguous", "type": "choice", "prediction": "cannot_determine", "probabilities": {"cannot_determine": 0.9851136207580566, "charge_data_team": 0.004645786713808775, "charge_platform_team": 0.004086314234882593, "split_50_50": 0.0061542801558971405}, "none_candidate_id": null, "predicted_none": false, "label": "cannot_determine", "correct": true, "confidence": 0.9851136207580566, "record_seconds": 0.38777492195367813, "batch_seconds": 0.38777492195367813, "batch_records": 1, "timing_scope": "record", "brier": 0.0002977607655338943, "nll": 0.014998293481767178} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-03", "group_id": "jevbench:hard-opus-b-multi_hop-03", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "tier3_cfo", "probabilities": {"tier1_team_lead": 0.15589994192123413, "tier2_department_head": 0.1972726285457611, "tier3_cfo": 0.5322976112365723, "tier4_board": 0.11452978849411011}, "none_candidate_id": null, "predicted_none": false, "label": "tier3_cfo", "correct": true, "confidence": 0.5322976112365723, "record_seconds": 2.104744452983141, "batch_seconds": 2.104744452983141, "batch_records": 1, "timing_scope": "record", "brier": 0.2950838804244995, "nll": 0.6305525302886963} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-04", "group_id": "jevbench:hard-opus-b-multi_hop-04", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "bjorn", "probabilities": {"ana": 0.08418291807174683, "bjorn": 0.4828298091888428, "chen": 0.0815255343914032, "dmitri": 0.06095448136329651, "esra": 0.16018573939800262, "no_page_ticket_only": 0.13032159209251404}, "none_candidate_id": null, "predicted_none": false, "label": "bjorn", "correct": true, "confidence": 0.4828298091888428, "record_seconds": 2.5746793057769537, "batch_seconds": 2.5746793057769537, "batch_records": 1, "timing_scope": "record", "brier": 0.3275568187236786, "nll": 0.7280910611152649} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-05", "group_id": "jevbench:hard-opus-b-multi_hop-05", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "score", "prediction": "1", "probabilities": {"0": 0.06419828534126282, "1": 0.4275071322917938, "2": 0.07298344373703003, "3": 0.38436761498451233, "4": 0.050943512469530106}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": false, "confidence": 0.4275071322917938, "record_seconds": 2.3437809450551867, "batch_seconds": 2.3437809450551867, "batch_records": 1, "timing_scope": "record", "brier": 1.1965771913528442, "nll": 2.617522716522217} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-07", "group_id": "jevbench:hard-opus-b-multi_hop-07", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "credit_50_percent", "probabilities": {"credit_10_percent": 0.04205353558063507, "credit_20_percent": 0.24088391661643982, "credit_25_percent": 0.2735840678215027, "credit_50_percent": 0.29575130343437195, "credit_full_month": 0.09467767179012299, "no_credit": 0.05304957926273346}, "none_candidate_id": null, "predicted_none": false, "label": "credit_20_percent", "correct": false, "confidence": 0.29575130343437195, "record_seconds": 2.813248820602894, "batch_seconds": 2.813248820602894, "batch_records": 1, "timing_scope": "record", "brier": 0.752120852470398, "nll": 1.4234400987625122} |
| {"record_id": "jevbench:hard-opus-b-multi_hop-08", "group_id": "jevbench:hard-opus-b-multi_hop-08", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "noul", "prediction": "false", "probabilities": {"false": 0.8654403686523438, "true": 0.13455958664417267}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.8654403686523438, "record_seconds": 0.8745089583098888, "batch_seconds": 0.8745089583098888, "batch_records": 1, "timing_scope": "record", "brier": 0.036212578415870667, "nll": 0.14451681077480316} |
| {"record_id": "jevbench:hard-opus-b-probability-01", "group_id": "jevbench:hard-opus-b-probability-01", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "merchant_wins", "probabilities": {"customer_wins": 0.26584258675575256, "customer_withdraws": 0.03060287795960903, "merchant_wins": 0.7035545110702515}, "none_candidate_id": null, "predicted_none": false, "label": "merchant_wins", "correct": true, "confidence": 0.7035545110702515, "record_seconds": 1.049927438609302, "batch_seconds": 1.049927438609302, "batch_records": 1, "timing_scope": "record", "brier": 0.007935035973787308, "nll": 0.9084252715110779} |
| {"record_id": "jevbench:hard-opus-b-probability-02", "group_id": "jevbench:hard-opus-b-probability-02", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "bad_push", "probabilities": {"bad_push": 0.8857632875442505, "database_hardware": 0.049404434859752655, "upstream_provider": 0.06483226269483566}, "none_candidate_id": null, "predicted_none": false, "label": "upstream_provider", "correct": false, "confidence": 0.8857632875442505, "record_seconds": 0.27369683887809515, "batch_seconds": 0.27369683887809515, "batch_records": 1, "timing_scope": "record", "brier": 0.8228119611740112, "nll": 2.253787040710449} |
| {"record_id": "jevbench:hard-opus-b-probability-03", "group_id": "jevbench:hard-opus-b-probability-03", "question_id": "decision", "source": "jevbench/hard/probability", "type": "noul", "prediction": "true", "probabilities": {"false": 0.28808292746543884, "true": 0.7119171023368835}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.7119171023368835, "record_seconds": 0.460681582801044, "batch_seconds": 0.460681582801044, "batch_records": 1, "timing_scope": "record", "brier": 0.0011923883575946093, "nll": 0.6225166320800781} |
| {"record_id": "jevbench:hard-opus-b-probability-04", "group_id": "jevbench:hard-opus-b-probability-04", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "attended", "probabilities": {"attended": 0.795161783695221, "late_cancellation": 0.04125271737575531, "no_show": 0.16358543932437897}, "none_candidate_id": null, "predicted_none": false, "label": "attended", "correct": true, "confidence": 0.795161783695221, "record_seconds": 0.3228919394314289, "batch_seconds": 0.3228919394314289, "batch_records": 1, "timing_scope": "record", "brier": 0.013211067765951157, "nll": 0.7107659578323364} |
| {"record_id": "jevbench:hard-opus-b-probability-06", "group_id": "jevbench:hard-opus-b-probability-06", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "overturned", "probabilities": {"modified": 0.0610431544482708, "overturned": 0.58465576171875, "upheld": 0.3543010652065277}, "none_candidate_id": null, "predicted_none": false, "label": "upheld", "correct": false, "confidence": 0.58465576171875, "record_seconds": 0.5488315615803003, "batch_seconds": 0.5488315615803003, "batch_records": 1, "timing_scope": "record", "brier": 0.22301813960075378, "nll": 1.0222995281219482} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-01", "group_id": "jevbench:hard-opus-b-tradeoff-01", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "prediction": "escalate_to_engineering", "probabilities": {"escalate_to_engineering": 0.5066345930099487, "escalate_to_security": 0.3860284984111786, "page_oncall": 0.006052038632333279, "process_refund": 0.02329069748520851, "send_kb_article": 0.07799414545297623}, "none_candidate_id": null, "predicted_none": false, "label": "escalate_to_engineering", "correct": true, "confidence": 0.5066345930099487, "record_seconds": 0.6915192045271397, "batch_seconds": 0.6915192045271397, "batch_records": 1, "timing_scope": "record", "brier": 0.39908960461616516, "nll": 0.6799652576446533} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-03", "group_id": "jevbench:hard-opus-b-tradeoff-03", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "prediction": "order_b", "probabilities": {"order_a": 0.21383434534072876, "order_b": 0.7077928185462952, "order_c": 0.039266180247068405, "order_d": 0.039106663316488266}, "none_candidate_id": null, "predicted_none": false, "label": "order_b", "correct": true, "confidence": 0.7077928185462952, "record_seconds": 0.5221585994586349, "batch_seconds": 0.5221585994586349, "batch_records": 1, "timing_scope": "record", "brier": 0.13418133556842804, "nll": 0.3456038534641266} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-06", "group_id": "jevbench:hard-opus-b-tradeoff-06", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "noul", "prediction": "true", "probabilities": {"false": 0.43883252143859863, "true": 0.5611674785614014}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.5611674785614014, "record_seconds": 0.30266179237514734, "batch_seconds": 0.30266179237514734, "batch_records": 1, "timing_scope": "record", "brier": 0.38514795899391174, "nll": 0.5777359008789062} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-07", "group_id": "jevbench:hard-opus-b-tradeoff-07", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "prediction": "p0_fix_24h", "probabilities": {"out_of_scope": 0.004366428125649691, "p0_fix_24h": 0.5541700720787048, "p1_fix_7d": 0.41846904158592224, "p2_fix_30d": 0.012944168411195278, "p3_backlog": 0.010050301440060139}, "none_candidate_id": null, "predicted_none": false, "label": "p0_fix_24h", "correct": true, "confidence": 0.5541700720787048, "record_seconds": 0.5717803230509162, "batch_seconds": 0.5717803230509162, "batch_records": 1, "timing_scope": "record", "brier": 0.3741682767868042, "nll": 0.5902836322784424} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-08", "group_id": "jevbench:hard-opus-b-tradeoff-08", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "noul", "prediction": "true", "probabilities": {"false": 0.4981672763824463, "true": 0.5018326640129089}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.5018326640129089, "record_seconds": 0.23064491525292397, "batch_seconds": 0.23064491525292397, "batch_records": 1, "timing_scope": "record", "brier": 0.4963413178920746, "nll": 0.6894885301589966} |
| {"record_id": "jevbench:hard-opus-b-tradeoff-12", "group_id": "jevbench:hard-opus-b-tradeoff-12", "question_id": "decision", "source": "jevbench/hard/tradeoff", "type": "choice", "prediction": "refund_now", "probabilities": {"deny_refund": 0.01703101024031639, "escalate_to_fraud_team": 0.0040468014776706696, "offer_store_credit": 0.01800941862165928, "refund_now": 0.9339783787727356, "wait_for_inspection": 0.026934396475553513}, "none_candidate_id": null, "predicted_none": false, "label": "refund_now", "correct": true, "confidence": 0.9339783787727356, "record_seconds": 0.47752173710614443, "batch_seconds": 0.47752173710614443, "batch_records": 1, "timing_scope": "record", "brier": 0.0057150875218212605, "nll": 0.06830199062824249} |
| {"record_id": "jevbench:hard-opus-c-long_policy-02", "group_id": "jevbench:hard-opus-c-long_policy-02", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "erase_but_retain_transaction_records", "probabilities": {"erase_all": 0.12692327797412872, "erase_but_retain_transaction_records": 0.7881578207015991, "erase_marketing_only": 0.051998961716890335, "reject_unverified_request": 0.018443038687109947, "restrict_all_pending_legal_hold": 0.014476931653916836}, "none_candidate_id": null, "predicted_none": false, "label": "erase_but_retain_transaction_records", "correct": true, "confidence": 0.7881578207015991, "record_seconds": 3.3116405867040157, "batch_seconds": 3.3116405867040157, "batch_records": 1, "timing_scope": "record", "brier": 0.06424024701118469, "nll": 0.23805692791938782} |
| {"record_id": "jevbench:hard-opus-c-long_policy-03", "group_id": "jevbench:hard-opus-c-long_policy-03", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "eur_150", "probabilities": {"eur_0": 0.11986807733774185, "eur_150": 0.4540376365184784, "eur_200": 0.1723472774028778, "eur_300": 0.11051394790410995, "eur_400": 0.08913978189229965, "eur_600": 0.054093316197395325}, "none_candidate_id": null, "predicted_none": false, "label": "eur_400", "correct": false, "confidence": 0.4540376365184784, "record_seconds": 3.803407296538353, "batch_seconds": 3.803407296538353, "batch_records": 1, "timing_scope": "record", "brier": 1.095028042793274, "nll": 2.4175496101379395} |
| {"record_id": "jevbench:hard-opus-c-long_policy-04", "group_id": "jevbench:hard-opus-c-long_policy-04", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.2730410099029541, "true": 0.7269589900970459}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.7269589900970459, "record_seconds": 1.2567162858322263, "batch_seconds": 1.2567162858322263, "batch_records": 1, "timing_scope": "record", "brier": 0.14910279214382172, "nll": 0.3188852071762085} |
| {"record_id": "jevbench:hard-opus-c-long_policy-05", "group_id": "jevbench:hard-opus-c-long_policy-05", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "score", "prediction": "2", "probabilities": {"0": 0.2513563632965088, "1": 0.21909388899803162, "2": 0.39178889989852905, "3": 0.13776081800460815}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": false, "confidence": 0.39178889989852905, "record_seconds": 2.364857361651957, "batch_seconds": 2.364857361651957, "batch_records": 1, "timing_scope": "record", "brier": 0.8454709053039551, "nll": 1.5182548761367798} |
| {"record_id": "jevbench:hard-opus-c-long_policy-08", "group_id": "jevbench:hard-opus-c-long_policy-08", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "score", "prediction": "2", "probabilities": {"0": 0.02004903368651867, "1": 0.11448387056589127, "2": 0.8335720896720886, "3": 0.031895026564598083}, "none_candidate_id": null, "predicted_none": false, "label": "1", "correct": false, "confidence": 0.8335720896720886, "record_seconds": 2.309033495374024, "batch_seconds": 2.309033495374024, "batch_records": 1, "timing_scope": "record", "brier": 1.480400562286377, "nll": 2.1673214435577393} |
| {"record_id": "jevbench:hard-opus-c-long_policy-10", "group_id": "jevbench:hard-opus-c-long_policy-10", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "credit_minus_restocking_fee", "probabilities": {"credit_minus_restocking_fee": 0.38541677594184875, "full_credit": 0.056241679936647415, "reject_return": 0.34301117062568665, "repair_or_replace_only": 0.2153303474187851}, "none_candidate_id": null, "predicted_none": false, "label": "credit_minus_restocking_fee", "correct": true, "confidence": 0.38541677594184875, "record_seconds": 2.733581183478236, "batch_seconds": 2.733581183478236, "batch_records": 1, "timing_scope": "record", "brier": 0.5448995232582092, "nll": 0.9534299969673157} |
| {"record_id": "jevbench:hard-opus-c-long_policy-11", "group_id": "jevbench:hard-opus-c-long_policy-11", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "noul", "prediction": "true", "probabilities": {"false": 0.20829598605632782, "true": 0.7917040586471558}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.7917040586471558, "record_seconds": 1.3080795565620065, "batch_seconds": 1.3080795565620065, "batch_records": 1, "timing_scope": "record", "brier": 1.2535905838012695, "nll": 1.5687952041625977} |
| {"record_id": "jevbench:hard-opus-c-probability-03", "group_id": "jevbench:hard-opus-c-probability-03", "question_id": "decision", "source": "jevbench/hard/probability", "type": "choice", "prediction": "kuznets_technik", "probabilities": {"halvorsen_polymer": 0.11481134593486786, "kuznets_technik": 0.7429253458976746, "ortega_seal_works": 0.1422632783651352}, "none_candidate_id": null, "predicted_none": false, "label": "kuznets_technik", "correct": true, "confidence": 0.7429253458976746, "record_seconds": 0.2981020836159587, "batch_seconds": 0.2981020836159587, "batch_records": 1, "timing_scope": "record", "brier": 0.02975812368094921, "nll": 0.995969295501709} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-02", "group_id": "jevbench:hard-opus-c-temporal_numeric-02", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "noul", "prediction": "false", "probabilities": {"false": 0.7795811295509338, "true": 0.22041884064674377}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": false, "confidence": 0.7795811295509338, "record_seconds": 0.345874116756022, "batch_seconds": 0.345874116756022, "batch_records": 1, "timing_scope": "record", "brier": 1.2154935598373413, "nll": 1.5122257471084595} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-03", "group_id": "jevbench:hard-opus-c-temporal_numeric-03", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "eur_617_00", "probabilities": {"eur_617_00": 0.23082797229290009, "eur_688_00": 0.18349704146385193, "eur_692_00": 0.20555393397808075, "eur_693_90": 0.21665559709072113, "eur_812_00": 0.1634654551744461}, "none_candidate_id": null, "predicted_none": false, "label": "eur_692_00", "correct": false, "confidence": 0.23082797229290009, "record_seconds": 0.6915720049291849, "batch_seconds": 0.6915720049291849, "batch_records": 1, "timing_scope": "record", "brier": 0.7917578816413879, "nll": 1.5820468664169312} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-04", "group_id": "jevbench:hard-opus-c-temporal_numeric-04", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "usd_2303_01", "probabilities": {"usd_2222_50": 0.16741016507148743, "usd_2303_01": 0.25712862610816956, "usd_2303_33": 0.24703462421894073, "usd_2335_00": 0.18136240541934967, "usd_2340_00": 0.14706414937973022}, "none_candidate_id": null, "predicted_none": false, "label": "usd_2335_00", "correct": false, "confidence": 0.25712862610816956, "record_seconds": 0.581280623562634, "batch_seconds": 0.581280623562634, "batch_records": 1, "timing_scope": "record", "brier": 0.8469628095626831, "nll": 1.7072579860687256} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-06", "group_id": "jevbench:hard-opus-c-temporal_numeric-06", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "sep_16_taxi", "probabilities": {"none_declined": 0.1567889004945755, "sep_15_office_chair": 0.21037353575229645, "sep_16_taxi": 0.3094007074832916, "sep_17_parking": 0.06572379171848297, "sep_18_book_set": 0.09267035871744156, "sep_21_client_dinner": 0.1650427281856537}, "none_candidate_id": null, "predicted_none": false, "label": "sep_18_book_set", "correct": false, "confidence": 0.3094007074832916, "record_seconds": 0.9149541249498725, "batch_seconds": 0.9149541249498725, "batch_records": 1, "timing_scope": "record", "brier": 1.0193743705749512, "nll": 2.378706693649292} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-08", "group_id": "jevbench:hard-opus-c-temporal_numeric-08", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "eur_532_01", "probabilities": {"eur_497_40": 0.13802433013916016, "eur_498_81": 0.17646881937980652, "eur_506_06": 0.20895133912563324, "eur_513_69": 0.2273845374584198, "eur_532_01": 0.24917103350162506}, "none_candidate_id": null, "predicted_none": false, "label": "eur_498_81", "correct": false, "confidence": 0.24917103350162506, "record_seconds": 0.7919412646442652, "batch_seconds": 0.7919412646442652, "batch_records": 1, "timing_scope": "record", "brier": 0.8547049164772034, "nll": 1.7346110343933105} |
| {"record_id": "jevbench:hard-opus-c-temporal_numeric-12", "group_id": "jevbench:hard-opus-c-temporal_numeric-12", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "expired_24_month_term", "probabilities": {"covered": 0.32376694679260254, "expired_24_month_term": 0.34761133790016174, "expired_30_month_cap": 0.32862168550491333}, "none_candidate_id": null, "predicted_none": false, "label": "expired_30_month_cap", "correct": false, "confidence": 0.34761133790016174, "record_seconds": 0.43693527206778526, "batch_seconds": 0.43693527206778526, "batch_records": 1, "timing_scope": "record", "brier": 0.676407516002655, "nll": 1.1128480434417725} |
| {"record_id": "jevbench:hard-sol-a-adversarial-01", "group_id": "jevbench:hard-sol-a-adversarial-01", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "choice", "prediction": "deny_missing_approval", "probabilities": {"deny_inactive_user": 0.003249126486480236, "deny_missing_approval": 0.9756714105606079, "grant_access": 0.012553084641695023, "grant_owner_exception": 0.008526366204023361}, "none_candidate_id": null, "predicted_none": false, "label": "deny_missing_approval", "correct": true, "confidence": 0.9756714105606079, "record_seconds": 0.17959998734295368, "batch_seconds": 0.17959998734295368, "batch_records": 1, "timing_scope": "record", "brier": 0.0008327160030603409, "nll": 0.024629419669508934} |
| {"record_id": "jevbench:hard-sol-a-adversarial-06", "group_id": "jevbench:hard-sol-a-adversarial-06", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9562610387802124, "true": 0.04373887926340103}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9562610387802124, "record_seconds": 0.09913669712841511, "batch_seconds": 0.09913669712841511, "batch_records": 1, "timing_scope": "record", "brier": 0.0038261862937361, "nll": 0.044724348932504654} |
| {"record_id": "jevbench:hard-sol-a-adversarial-07", "group_id": "jevbench:hard-sol-a-adversarial-07", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "choice", "prediction": "retain_until_2031", "probabilities": {"delete_now": 0.04994026571512222, "needs_signature_review": 0.005381946451961994, "retain_indefinitely": 0.032038331031799316, "retain_until_2031": 0.9126393795013428}, "none_candidate_id": null, "predicted_none": false, "label": "retain_until_2031", "correct": true, "confidence": 0.9126393795013428, "record_seconds": 0.18434704747051, "batch_seconds": 0.18434704747051, "batch_records": 1, "timing_scope": "record", "brier": 0.011181328445672989, "nll": 0.09141445904970169} |
| {"record_id": "jevbench:hard-sol-a-adversarial-08", "group_id": "jevbench:hard-sol-a-adversarial-08", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "noul", "prediction": "true", "probabilities": {"false": 0.028647441416978836, "true": 0.9713525176048279}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9713525176048279, "record_seconds": 0.09586076438426971, "batch_seconds": 0.09586076438426971, "batch_records": 1, "timing_scope": "record", "brier": 0.0016413541743531823, "nll": 0.029065830633044243} |
| {"record_id": "jevbench:hard-sol-a-adversarial-09", "group_id": "jevbench:hard-sol-a-adversarial-09", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "score", "prediction": "2", "probabilities": {"0": 0.007854649797081947, "1": 0.025495538488030434, "2": 0.9574641585350037, "3": 0.00918556097894907}, "none_candidate_id": null, "predicted_none": false, "label": "2", "correct": true, "confidence": 0.9574641585350037, "record_seconds": 0.15904368087649345, "batch_seconds": 0.15904368087649345, "batch_records": 1, "timing_scope": "record", "brier": 0.0026053902693092823, "nll": 0.04346699267625809} |
| {"record_id": "jevbench:hard-sol-a-adversarial-11", "group_id": "jevbench:hard-sol-a-adversarial-11", "question_id": "decision", "source": "jevbench/hard/adversarial", "type": "choice", "prediction": "standard_research", "probabilities": {"ethics_review": 0.007739677093923092, "major_grants": 0.004017457365989685, "manual_hold": 0.004262182395905256, "standard_research": 0.9839805960655212}, "none_candidate_id": null, "predicted_none": false, "label": "standard_research", "correct": true, "confidence": 0.9839805960655212, "record_seconds": 0.19578189495950937, "batch_seconds": 0.19578189495950937, "batch_records": 1, "timing_scope": "record", "brier": 0.00035083008697256446, "nll": 0.01614910177886486} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-01", "group_id": "jevbench:hard-sol-a-multi_hop-01", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "accept_s2_dusk", "probabilities": {"accept_s2_dusk": 0.7076132297515869, "reject_cutoff": 0.05548733472824097, "reject_tier": 0.06180526688694954, "reroute_s3": 0.1750941127538681}, "none_candidate_id": null, "predicted_none": false, "label": "accept_s2_dusk", "correct": true, "confidence": 0.7076132297515869, "record_seconds": 1.58099433965981, "batch_seconds": 1.58099433965981, "batch_records": 1, "timing_scope": "record", "brier": 0.12304670363664627, "nll": 0.3458576202392578} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-02", "group_id": "jevbench:hard-sol-a-multi_hop-02", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "security_primary", "probabilities": {"executive_bridge": 0.03540295735001564, "heron_q2": 0.016338735818862915, "insufficient_information": 0.10315712541341782, "security_primary": 0.8451011776924133}, "none_candidate_id": null, "predicted_none": false, "label": "security_primary", "correct": true, "confidence": 0.8451011776924133, "record_seconds": 1.5999326696619391, "batch_seconds": 1.5999326696619391, "batch_records": 1, "timing_scope": "record", "brier": 0.03615536168217659, "nll": 0.16829891502857208} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-05", "group_id": "jevbench:hard-sol-a-multi_hop-05", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "require_director", "probabilities": {"pay_manager_approved": 0.1867215633392334, "reject_category": 0.05453871563076973, "require_director": 0.7027518153190613, "require_finance_board": 0.0559878796339035}, "none_candidate_id": null, "predicted_none": false, "label": "require_director", "correct": true, "confidence": 0.7027518153190613, "record_seconds": 1.5323570063337684, "batch_seconds": 1.5323570063337684, "batch_records": 1, "timing_scope": "record", "brier": 0.12933053076267242, "nll": 0.3527514934539795} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-07", "group_id": "jevbench:hard-sol-a-multi_hop-07", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "admit", "probabilities": {"admit": 0.8561353087425232, "deny_prerequisite": 0.07171820104122162, "deny_score": 0.05681934952735901, "manual_equivalency": 0.015327139757573605}, "none_candidate_id": null, "predicted_none": false, "label": "admit", "correct": true, "confidence": 0.8561353087425232, "record_seconds": 1.532513097859919, "batch_seconds": 1.532513097859919, "batch_records": 1, "timing_scope": "record", "brier": 0.029303910210728645, "nll": 0.15532684326171875} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-08", "group_id": "jevbench:hard-sol-a-multi_hop-08", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "local_maintenance", "probabilities": {"central_engineering": 0.28785282373428345, "insufficient_log": 0.05759218707680702, "local_maintenance": 0.6474338173866272, "safety_shutdown": 0.007121166680008173}, "none_candidate_id": null, "predicted_none": false, "label": "central_engineering", "correct": false, "confidence": 0.6474338173866272, "record_seconds": 1.4957067733630538, "batch_seconds": 1.4957067733630538, "batch_records": 1, "timing_scope": "record", "brier": 0.9296917915344238, "nll": 1.2453060150146484} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-09", "group_id": "jevbench:hard-sol-a-multi_hop-09", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "require_steward", "probabilities": {"confirm_l3": 0.03954703360795975, "reject_capacity": 0.03819508105516434, "require_steward": 0.8813513517379761, "switch_l4": 0.040906600654125214}, "none_candidate_id": null, "predicted_none": false, "label": "require_steward", "correct": true, "confidence": 0.8813513517379761, "record_seconds": 1.4935943661257625, "batch_seconds": 1.4935943661257625, "batch_records": 1, "timing_scope": "record", "brier": 0.01877368427813053, "nll": 0.1262989193201065} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-10", "group_id": "jevbench:hard-sol-a-multi_hop-10", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "approve_45_days", "probabilities": {"approve_45_days": 0.5696258544921875, "deny_sensitive": 0.07149305194616318, "needs_field_review": 0.08338889479637146, "reduce_to_30_days": 0.2754921615123749}, "none_candidate_id": null, "predicted_none": false, "label": "approve_45_days", "correct": true, "confidence": 0.5696258544921875, "record_seconds": 1.5528358481824398, "batch_seconds": 1.5528358481824398, "batch_records": 1, "timing_scope": "record", "brier": 0.27318280935287476, "nll": 0.5627755522727966} |
| {"record_id": "jevbench:hard-sol-a-multi_hop-12", "group_id": "jevbench:hard-sol-a-multi_hop-12", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "needs_authentication", "probabilities": {"covered_recurrence": 0.21151623129844666, "expired": 0.09989652037620544, "needs_authentication": 0.6524174809455872, "wrong_issue_class": 0.036169834434986115}, "none_candidate_id": null, "predicted_none": false, "label": "needs_authentication", "correct": true, "confidence": 0.6524174809455872, "record_seconds": 1.4981071585789323, "batch_seconds": 1.4981071585789323, "batch_records": 1, "timing_scope": "record", "brier": 0.17684030532836914, "nll": 0.42707061767578125} |
| {"record_id": "jevbench:hard-sol-a-trap-02", "group_id": "jevbench:hard-sol-a-trap-02", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9797027707099915, "true": 0.02029719576239586}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9797027707099915, "record_seconds": 0.11348837707191706, "batch_seconds": 0.11348837707191706, "batch_records": 1, "timing_scope": "record", "brier": 0.0008239536546170712, "nll": 0.02050604857504368} |
| {"record_id": "jevbench:hard-sol-a-trap-04", "group_id": "jevbench:hard-sol-a-trap-04", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9535537958145142, "true": 0.04644622653722763}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9535537958145142, "record_seconds": 0.09647584240883589, "batch_seconds": 0.09647584240883589, "batch_records": 1, "timing_scope": "record", "brier": 0.004314501769840717, "nll": 0.04755943641066551} |
| {"record_id": "jevbench:hard-sol-a-trap-05", "group_id": "jevbench:hard-sol-a-trap-05", "question_id": "decision", "source": "jevbench/hard/trap", "type": "score", "prediction": "0", "probabilities": {"0": 0.7097122073173523, "1": 0.17697776854038239, "2": 0.10271372646093369, "3": 0.010596252977848053}, "none_candidate_id": null, "predicted_none": false, "label": "0", "correct": true, "confidence": 0.7097122073173523, "record_seconds": 0.23851874843239784, "batch_seconds": 0.23851874843239784, "batch_records": 1, "timing_scope": "record", "brier": 0.1262505203485489, "nll": 0.3428957462310791} |
| {"record_id": "jevbench:hard-sol-a-trap-06", "group_id": "jevbench:hard-sol-a-trap-06", "question_id": "decision", "source": "jevbench/hard/trap", "type": "choice", "prediction": "keep_open_unresolved", "probabilities": {"await_internal_metrics": 0.02525196224451065, "close_agent_verified": 0.006857440806925297, "close_confirmed": 0.007065389305353165, "keep_open_unresolved": 0.9608251452445984}, "none_candidate_id": null, "predicted_none": false, "label": "keep_open_unresolved", "correct": true, "confidence": 0.9608251452445984, "record_seconds": 0.23190024867653847, "batch_seconds": 0.23190024867653847, "batch_records": 1, "timing_scope": "record", "brier": 0.0022692750208079815, "nll": 0.03996283560991287} |
| {"record_id": "jevbench:hard-sol-a-trap-08", "group_id": "jevbench:hard-sol-a-trap-08", "question_id": "decision", "source": "jevbench/hard/trap", "type": "choice", "prediction": "late", "probabilities": {"canceled": 0.0026051527820527554, "indeterminate": 0.003745220834389329, "late": 0.9903332591056824, "on_time": 0.00331637030467391}, "none_candidate_id": null, "predicted_none": false, "label": "late", "correct": true, "confidence": 0.9903332591056824, "record_seconds": 0.17708178143948317, "batch_seconds": 0.17708178143948317, "batch_records": 1, "timing_scope": "record", "brier": 0.00012525769125204533, "nll": 0.009713767096400261} |
| {"record_id": "jevbench:hard-sol-a-trap-10", "group_id": "jevbench:hard-sol-a-trap-10", "question_id": "decision", "source": "jevbench/hard/trap", "type": "choice", "prediction": "blocked_second_review", "probabilities": {"blocked_failed_test": 0.005154828075319529, "blocked_second_review": 0.9651814103126526, "ready_one_review": 0.019851887598633766, "ready_two_reviews": 0.0098118232563138}, "none_candidate_id": null, "predicted_none": false, "label": "blocked_second_review", "correct": true, "confidence": 0.9651814103126526, "record_seconds": 0.12806829065084457, "batch_seconds": 0.12806829065084457, "batch_records": 1, "timing_scope": "record", "brier": 0.0017292756820097566, "nll": 0.035439204424619675} |
| {"record_id": "jevbench:hard-sol-a-trap-13", "group_id": "jevbench:hard-sol-a-trap-13", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9507079720497131, "true": 0.049292031675577164}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9507079720497131, "record_seconds": 0.09634083043783903, "batch_seconds": 0.09634083043783903, "batch_records": 1, "timing_scope": "record", "brier": 0.004859408363699913, "nll": 0.050548337399959564} |
| {"record_id": "jevbench:hard-sol-a-trap-15", "group_id": "jevbench:hard-sol-a-trap-15", "question_id": "decision", "source": "jevbench/hard/trap", "type": "noul", "prediction": "false", "probabilities": {"false": 0.5587431192398071, "true": 0.44125691056251526}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.5587431192398071, "record_seconds": 0.12879366893321276, "batch_seconds": 0.12879366893321276, "batch_records": 1, "timing_scope": "record", "brier": 0.38941529393196106, "nll": 0.5820654630661011} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-01", "group_id": "jevbench:hard-sol-b-judge_hard-01", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.04421575367450714, "true": 0.955784261226654}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.955784261226654, "record_seconds": 0.10846233274787664, "batch_seconds": 0.10846233274787664, "batch_records": 1, "timing_scope": "record", "brier": 0.003910064697265625, "nll": 0.04522306099534035} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-02", "group_id": "jevbench:hard-sol-b-judge_hard-02", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.0640333890914917, "true": 0.9359666109085083}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.9359666109085083, "record_seconds": 0.09648764785379171, "batch_seconds": 0.09648764785379171, "batch_records": 1, "timing_scope": "record", "brier": 1.752066969871521, "nll": 2.7483506202697754} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-05", "group_id": "jevbench:hard-sol-b-judge_hard-05", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.04492334648966789, "true": 0.9550766348838806}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9550766348838806, "record_seconds": 0.09702091477811337, "batch_seconds": 0.09702091477811337, "batch_records": 1, "timing_scope": "record", "brier": 0.00403621606528759, "nll": 0.045963697135448456} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-08", "group_id": "jevbench:hard-sol-b-judge_hard-08", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.04683755710721016, "true": 0.953162431716919}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.953162431716919, "record_seconds": 0.09759537409991026, "batch_seconds": 0.09759537409991026, "batch_records": 1, "timing_scope": "record", "brier": 1.8170372247695923, "nll": 3.061069965362549} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-10", "group_id": "jevbench:hard-sol-b-judge_hard-10", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.2538122236728668, "true": 0.7461877465248108}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.7461877465248108, "record_seconds": 0.0968489870429039, "batch_seconds": 0.0968489870429039, "batch_records": 1, "timing_scope": "record", "brier": 1.1135923862457275, "nll": 1.3711605072021484} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-14", "group_id": "jevbench:hard-sol-b-judge_hard-14", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "false", "probabilities": {"false": 0.9451336860656738, "true": 0.05486626923084259}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.9451336860656738, "record_seconds": 0.09886821638792753, "batch_seconds": 0.09886821638792753, "batch_records": 1, "timing_scope": "record", "brier": 0.006020619533956051, "nll": 0.05642889440059662} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-15", "group_id": "jevbench:hard-sol-b-judge_hard-15", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.01959487609565258, "true": 0.9804051518440247}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9804051518440247, "record_seconds": 0.09838079567998648, "batch_seconds": 0.09838079567998648, "batch_records": 1, "timing_scope": "record", "brier": 0.0007679172558709979, "nll": 0.019789371639490128} |
| {"record_id": "jevbench:hard-sol-b-judge_hard-18", "group_id": "jevbench:hard-sol-b-judge_hard-18", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "false", "probabilities": {"false": 0.6118301153182983, "true": 0.3881698548793793}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.6118301153182983, "record_seconds": 0.09708203375339508, "batch_seconds": 0.09708203375339508, "batch_records": 1, "timing_scope": "record", "brier": 0.3013516962528229, "nll": 0.4913006126880646} |
| {"record_id": "jevbench:hard-sol-b-long_policy-01", "group_id": "jevbench:hard-sol-b-long_policy-01", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "approve_meals_only", "probabilities": {"approve_all": 0.06832686811685562, "approve_lodging_only": 0.2905815839767456, "approve_meals_only": 0.4671134948730469, "reject_entire_claim": 0.1739780604839325}, "none_candidate_id": null, "predicted_none": false, "label": "reject_entire_claim", "correct": false, "confidence": 0.4671134948730469, "record_seconds": 1.997850532643497, "batch_seconds": 1.997850532643497, "batch_records": 1, "timing_scope": "record", "brier": 0.98961341381073, "nll": 1.748826026916504} |
| {"record_id": "jevbench:hard-sol-b-long_policy-02", "group_id": "jevbench:hard-sol-b-long_policy-02", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "deny_current_export", "probabilities": {"analytics_owner_only": 0.026962408795952797, "deny_current_export": 0.8584886789321899, "privacy_and_security": 0.06186803802847862, "privacy_only": 0.0526808500289917}, "none_candidate_id": null, "predicted_none": false, "label": "deny_current_export", "correct": true, "confidence": 0.8584886789321899, "record_seconds": 1.9080701498314738, "batch_seconds": 1.9080701498314738, "batch_records": 1, "timing_scope": "record", "brier": 0.02735535241663456, "nll": 0.15258178114891052} |
| {"record_id": "jevbench:hard-sol-b-long_policy-05", "group_id": "jevbench:hard-sol-b-long_policy-05", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "delete_profile_omar_april_and_table", "probabilities": {"delete_everything": 0.03819653019309044, "delete_nothing": 0.0581112839281559, "delete_only_profile": 0.37992578744888306, "delete_profile_omar_april_and_table": 0.5237663984298706}, "none_candidate_id": null, "predicted_none": false, "label": "delete_profile_omar_april_and_table", "correct": true, "confidence": 0.5237663984298706, "record_seconds": 1.8603875003755093, "batch_seconds": 1.8603875003755093, "batch_records": 1, "timing_scope": "record", "brier": 0.37597793340682983, "nll": 0.6467095017433167} |
| {"record_id": "jevbench:hard-sol-b-long_policy-06", "group_id": "jevbench:hard-sol-b-long_policy-06", "question_id": "decision", "source": "jevbench/hard/long_policy", "type": "choice", "prediction": "river_arts", "probabilities": {"east_ward": 0.34771913290023804, "harbor_neighbors": 0.04282234236598015, "no_award": 0.18511854112148285, "river_arts": 0.42433997988700867}, "none_candidate_id": null, "predicted_none": false, "label": "east_ward", "correct": false, "confidence": 0.42433997988700867, "record_seconds": 1.8670606976374984, "batch_seconds": 1.8670606976374984, "batch_records": 1, "timing_scope": "record", "brier": 0.64163738489151, "nll": 1.0563602447509766} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-01", "group_id": "jevbench:hard-sol-b-routing_hard-01", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "office_file_repair", "probabilities": {"data_pipeline": 0.006566003896296024, "desktop_support": 0.009571585804224014, "finance_review": 0.024683644995093346, "office_file_repair": 0.893677294254303, "spreadsheet_formula": 0.06550146639347076}, "none_candidate_id": null, "predicted_none": false, "label": "office_file_repair", "correct": true, "confidence": 0.893677294254303, "record_seconds": 0.20176758617162704, "batch_seconds": 0.20176758617162704, "batch_records": 1, "timing_scope": "record", "brier": 0.016338970512151718, "nll": 0.1124105378985405} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-02", "group_id": "jevbench:hard-sol-b-routing_hard-02", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "ledger_reconciliation", "probabilities": {"customer_support": 0.010846778750419617, "fraud_review": 0.004331686533987522, "ledger_reconciliation": 0.96827232837677, "mobile_app": 0.009829487651586533, "payment_disputes": 0.0067197540774941444}, "none_candidate_id": null, "predicted_none": false, "label": "ledger_reconciliation", "correct": true, "confidence": 0.96827232837677, "record_seconds": 0.19556577689945698, "batch_seconds": 0.19556577689945698, "batch_records": 1, "timing_scope": "record", "brier": 0.0012848351616412401, "nll": 0.03224189952015877} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-03", "group_id": "jevbench:hard-sol-b-routing_hard-03", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "network_dns", "probabilities": {"api_platform": 0.005023708567023277, "authoritative_dns": 0.005838404409587383, "endpoint_devices": 0.010977823287248611, "network_dns": 0.9738848805427551, "tls_certificates": 0.004275203216820955}, "none_candidate_id": null, "predicted_none": false, "label": "network_dns", "correct": true, "confidence": 0.9738848805427551, "record_seconds": 0.19772124290466309, "batch_seconds": 0.19772124290466309, "batch_records": 1, "timing_scope": "record", "brier": 0.0008801140356808901, "nll": 0.026462174952030182} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-07", "group_id": "jevbench:hard-sol-b-routing_hard-07", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "release_notarization", "probabilities": {"ci_infrastructure": 0.004994353279471397, "code_signing": 0.014546087011694908, "endpoint_security": 0.005438716616481543, "macos_runtime": 0.006078985054045916, "release_notarization": 0.9689418077468872}, "none_candidate_id": null, "predicted_none": false, "label": "release_notarization", "correct": true, "confidence": 0.9689418077468872, "record_seconds": 0.1782538304105401, "batch_seconds": 0.1782538304105401, "batch_records": 1, "timing_scope": "record", "brier": 0.0012676772894337773, "nll": 0.03155072405934334} |
| {"record_id": "jevbench:hard-sol-b-routing_hard-09", "group_id": "jevbench:hard-sol-b-routing_hard-09", "question_id": "decision", "source": "jevbench/hard/routing_hard", "type": "choice", "prediction": "source_access", "probabilities": {"git_client": 0.010141363367438316, "identity_login": 0.004429670982062817, "repository_admin": 0.012433690950274467, "source_access": 0.966783881187439, "vpn_network": 0.006211461033672094}, "none_candidate_id": null, "predicted_none": false, "label": "source_access", "correct": true, "confidence": 0.966783881187439, "record_seconds": 0.17888979334384203, "batch_seconds": 0.17888979334384203, "batch_records": 1, "timing_scope": "record", "brier": 0.0014189586509019136, "nll": 0.0337803028523922} |
| {"record_id": "jevbench:hard-sol-b-temporal_numeric-01", "group_id": "jevbench:hard-sol-b-temporal_numeric-01", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "within_window", "probabilities": {"before_open": 0.05449487641453743, "late_by_2h_or_more": 0.2456919103860855, "late_by_under_2h": 0.2401358187198639, "within_window": 0.4596773684024811}, "none_candidate_id": null, "predicted_none": false, "label": "late_by_under_2h", "correct": false, "confidence": 0.4596773684024811, "record_seconds": 0.17783670965582132, "batch_seconds": 0.17783670965582132, "batch_records": 1, "timing_scope": "record", "brier": 0.8520311117172241, "nll": 1.4265506267547607} |
| {"record_id": "jevbench:hard-sol-b-temporal_numeric-02", "group_id": "jevbench:hard-sol-b-temporal_numeric-02", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "over_limit", "probabilities": {"at_least_100gb_below": 0.08398865908384323, "exactly_equal": 0.14187556505203247, "over_limit": 0.5210461616516113, "under_100gb_below": 0.2530896067619324}, "none_candidate_id": null, "predicted_none": false, "label": "over_limit", "correct": true, "confidence": 0.5210461616516113, "record_seconds": 0.18051286600530148, "batch_seconds": 0.18051286600530148, "batch_records": 1, "timing_scope": "record", "brier": 0.3206338882446289, "nll": 0.6519166231155396} |
| {"record_id": "jevbench:hard-sol-b-temporal_numeric-03", "group_id": "jevbench:hard-sol-b-temporal_numeric-03", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "6_credits", "probabilities": {"12_credits": 0.17836622893810272, "15_credits": 0.16751675307750702, "6_credits": 0.3484267294406891, "9_credits": 0.3056902587413788}, "none_candidate_id": null, "predicted_none": false, "label": "9_credits", "correct": false, "confidence": 0.3484267294406891, "record_seconds": 0.136412694118917, "batch_seconds": 0.136412694118917, "batch_records": 1, "timing_scope": "record", "brier": 0.6633435487747192, "nll": 1.1851829290390015} |
| {"record_id": "jevbench:hard-sol-b-temporal_numeric-04", "group_id": "jevbench:hard-sol-b-temporal_numeric-04", "question_id": "decision", "source": "jevbench/hard/temporal_numeric", "type": "choice", "prediction": "narrow_fail", "probabilities": {"clear_fail": 0.2663542926311493, "narrow_fail": 0.5338926911354065, "pass": 0.1997530162334442}, "none_candidate_id": null, "predicted_none": false, "label": "pass", "correct": false, "confidence": 0.5338926911354065, "record_seconds": 0.1301105422899127, "batch_seconds": 0.1301105422899127, "batch_records": 1, "timing_scope": "record", "brier": 0.9963811635971069, "nll": 1.6106735467910767} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-03", "group_id": "jevbench:hard-sol-c-judge_hard-03", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.16195830702781677, "true": 0.8380417227745056}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.8380417227745056, "record_seconds": 0.15724632423371077, "batch_seconds": 0.15724632423371077, "batch_records": 1, "timing_scope": "record", "brier": 1.404627799987793, "nll": 1.8204163312911987} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-05", "group_id": "jevbench:hard-sol-c-judge_hard-05", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.2744874358177185, "true": 0.7255125045776367}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.7255125045776367, "record_seconds": 0.13771894853562117, "batch_seconds": 0.13771894853562117, "batch_records": 1, "timing_scope": "record", "brier": 0.15068674087524414, "nll": 0.32087698578834534} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-07", "group_id": "jevbench:hard-sol-c-judge_hard-07", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.159246563911438, "true": 0.840753436088562}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.840753436088562, "record_seconds": 0.13148383609950542, "batch_seconds": 0.13148383609950542, "batch_records": 1, "timing_scope": "record", "brier": 1.413732647895813, "nll": 1.8373016119003296} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-08", "group_id": "jevbench:hard-sol-c-judge_hard-08", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.286298930644989, "true": 0.713701069355011}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": false, "confidence": 0.713701069355011, "record_seconds": 0.1299790358170867, "batch_seconds": 0.1299790358170867, "batch_records": 1, "timing_scope": "record", "brier": 1.0187383890151978, "nll": 1.2507188320159912} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-09", "group_id": "jevbench:hard-sol-c-judge_hard-09", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.10888597369194031, "true": 0.8911140561103821}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.8911140561103821, "record_seconds": 0.13410769030451775, "batch_seconds": 0.13410769030451775, "batch_records": 1, "timing_scope": "record", "brier": 0.02371230348944664, "nll": 0.11528284847736359} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-10", "group_id": "jevbench:hard-sol-c-judge_hard-10", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.07767130434513092, "true": 0.9223287105560303}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9223287105560303, "record_seconds": 0.15201884973794222, "batch_seconds": 0.15201884973794222, "batch_records": 1, "timing_scope": "record", "brier": 0.012065660208463669, "nll": 0.08085359632968903} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-11", "group_id": "jevbench:hard-sol-c-judge_hard-11", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "false", "probabilities": {"false": 0.976595401763916, "true": 0.023404555395245552}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.976595401763916, "record_seconds": 0.1339154141023755, "batch_seconds": 0.1339154141023755, "batch_records": 1, "timing_scope": "record", "brier": 0.0010955485049635172, "nll": 0.023682836443185806} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-13", "group_id": "jevbench:hard-sol-c-judge_hard-13", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "false", "probabilities": {"false": 0.5320716500282288, "true": 0.46792834997177124}, "none_candidate_id": null, "predicted_none": false, "label": "false", "correct": true, "confidence": 0.5320716500282288, "record_seconds": 0.13016830198466778, "batch_seconds": 0.13016830198466778, "batch_records": 1, "timing_scope": "record", "brier": 0.4379138946533203, "nll": 0.6309770941734314} |
| {"record_id": "jevbench:hard-sol-c-judge_hard-15", "group_id": "jevbench:hard-sol-c-judge_hard-15", "question_id": "decision", "source": "jevbench/hard/judge_hard", "type": "noul", "prediction": "true", "probabilities": {"false": 0.04237798601388931, "true": 0.9576219916343689}, "none_candidate_id": null, "predicted_none": false, "label": "true", "correct": true, "confidence": 0.9576219916343689, "record_seconds": 0.10891245771199465, "batch_seconds": 0.10891245771199465, "batch_records": 1, "timing_scope": "record", "brier": 0.003591789398342371, "nll": 0.04330215975642204} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-07", "group_id": "jevbench:hard-sol-c-multi_hop-07", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "programme_review", "probabilities": {"partial_release": 0.08476041257381439, "programme_review": 0.7315333485603333, "reject_draw": 0.06351010501384735, "release_draw": 0.12019610404968262}, "none_candidate_id": null, "predicted_none": false, "label": "programme_review", "correct": true, "confidence": 0.7315333485603333, "record_seconds": 1.7995190201327205, "batch_seconds": 1.7995190201327205, "batch_records": 1, "timing_scope": "record", "brier": 0.0977393090724945, "nll": 0.31261247396469116} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-09", "group_id": "jevbench:hard-sol-c-multi_hop-09", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "bus_substitution", "probabilities": {"bus_substitution": 0.8200205564498901, "call_fen_cross": 0.05034656450152397, "pass_fen_cross": 0.04889193922281265, "terminate_oakmere": 0.08074099570512772}, "none_candidate_id": null, "predicted_none": false, "label": "bus_substitution", "correct": true, "confidence": 0.8200205564498901, "record_seconds": 1.8691821293905377, "batch_seconds": 1.8691821293905377, "batch_records": 1, "timing_scope": "record", "brier": 0.04383690655231476, "nll": 0.19842587411403656} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-11", "group_id": "jevbench:hard-sol-c-multi_hop-11", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "coverage_excluded", "probabilities": {"adjuster_referral": 0.17458148300647736, "coverage_confirmed": 0.20167286694049835, "coverage_excluded": 0.3481858968734741, "documentation_needed": 0.2755597233772278}, "none_candidate_id": null, "predicted_none": false, "label": "coverage_excluded", "correct": true, "confidence": 0.3481858968734741, "record_seconds": 1.9304720032960176, "batch_seconds": 1.9304720032960176, "batch_records": 1, "timing_scope": "record", "brier": 0.5719454288482666, "nll": 1.0550187826156616} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-12", "group_id": "jevbench:hard-sol-c-multi_hop-12", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "conservation_hold", "probabilities": {"conservation_hold": 0.9398552775382996, "deny_request": 0.023340629413723946, "digital_copy_only": 0.0061012315563857555, "supervised_delivery": 0.03070288710296154}, "none_candidate_id": null, "predicted_none": false, "label": "conservation_hold", "correct": true, "confidence": 0.9398552775382996, "record_seconds": 0.36871440801769495, "batch_seconds": 0.36871440801769495, "batch_records": 1, "timing_scope": "record", "brier": 0.005142064765095711, "nll": 0.06202937662601471} |
| {"record_id": "jevbench:hard-sol-c-multi_hop-13", "group_id": "jevbench:hard-sol-c-multi_hop-13", "question_id": "decision", "source": "jevbench/hard/multi_hop", "type": "choice", "prediction": "normal_release", "probabilities": {"engineering_review": 0.022454895079135895, "normal_release": 0.9422183632850647, "rework": 0.01945585012435913, "scrap_lot": 0.015870900824666023}, "none_candidate_id": null, "predicted_none": false, "label": "normal_release", "correct": true, "confidence": 0.9422183632850647, "record_seconds": 1.8563760248944163, "batch_seconds": 1.8563760248944163, "batch_records": 1, "timing_scope": "record", "brier": 0.00447335559874773, "nll": 0.05951822176575661} |
|
|
