| { |
| "schema": "cortiq-model-card-comparison/1", |
| "model_file_sha256": "ed9b8ec2bbfe9e9fd30f14a5eaf82314f38bc7e7510a39772baa2de3801d79b1", |
| "model_bytes": 304520292, |
| "runtime": "cortiq 0.7.8", |
| "oracle_in_hybrid_benchmark": "deepseek/deepseek-v4.1-flash", |
| "benchmarks": [ |
| { |
| "id": "banking77", |
| "name": "BANKING77", |
| "n": 3080, |
| "labels": 77, |
| "local_correct": 2875, |
| "local_accuracy_pct": 93.34, |
| "jev_correct": 2636, |
| "jev_accuracy_pct": 85.58, |
| "local_wilson95_pct": [ |
| 92.41, |
| 94.17 |
| ], |
| "jev_wilson95_pct": [ |
| 84.3, |
| 86.78 |
| ], |
| "mcnemar_p": 9.72e-33, |
| "accepted": 2791, |
| "accepted_correct": 2714, |
| "accepted_accuracy_pct": 97.24, |
| "coverage_pct": 90.62, |
| "local_p50_ms": 3.92, |
| "local_p95_ms": 5.27, |
| "cortiq_https_p50_ms": 78.4, |
| "cortiq_https_p95_ms": 141.6, |
| "jev_http_p50_ms": 348.33, |
| "jev_http_p95_ms": 426.03, |
| "jev_latency_n": 3080, |
| "jev_latency_concurrency": 4, |
| "hybrid_correct": 2893, |
| "hybrid_accuracy_pct": 93.93, |
| "hybrid_oracle_calls": 282, |
| "hybrid_api_usd_per_million": 3.01, |
| "jev_api_usd_per_million": 183.69 |
| }, |
| { |
| "id": "clinc150", |
| "name": "CLINC150", |
| "n": 4500, |
| "labels": 150, |
| "local_correct": 4328, |
| "local_accuracy_pct": 96.18, |
| "jev_correct": 4354, |
| "jev_accuracy_pct": 96.76, |
| "local_wilson95_pct": [ |
| 95.58, |
| 96.7 |
| ], |
| "jev_wilson95_pct": [ |
| 96.2, |
| 97.23 |
| ], |
| "mcnemar_p": 0.099, |
| "accepted": 4145, |
| "accepted_correct": 4091, |
| "accepted_accuracy_pct": 98.7, |
| "coverage_pct": 92.11, |
| "local_p50_ms": 3.79, |
| "local_p95_ms": 4.57, |
| "cortiq_https_p50_ms": 78.5, |
| "cortiq_https_p95_ms": 134.6, |
| "jev_http_p50_ms": 391.51, |
| "jev_http_p95_ms": 528.49, |
| "jev_latency_n": 200, |
| "jev_latency_concurrency": 1, |
| "hybrid_correct": 4386, |
| "hybrid_accuracy_pct": 97.47, |
| "hybrid_oracle_calls": 351, |
| "hybrid_api_usd_per_million": 3.53, |
| "jev_api_usd_per_million": 271.61 |
| }, |
| { |
| "id": "massive", |
| "name": "MASSIVE", |
| "n": 2974, |
| "labels": 60, |
| "local_correct": 2562, |
| "local_accuracy_pct": 86.15, |
| "jev_correct": 2551, |
| "jev_accuracy_pct": 85.78, |
| "local_wilson95_pct": [ |
| 84.86, |
| 87.34 |
| ], |
| "jev_wilson95_pct": [ |
| 84.48, |
| 86.99 |
| ], |
| "mcnemar_p": 0.607, |
| "accepted": 1615, |
| "accepted_correct": 1581, |
| "accepted_accuracy_pct": 97.89, |
| "coverage_pct": 54.3, |
| "local_p50_ms": 3.0, |
| "local_p95_ms": 3.58, |
| "cortiq_https_p50_ms": 70.4, |
| "cortiq_https_p95_ms": 117.9, |
| "jev_http_p50_ms": 346.37, |
| "jev_http_p95_ms": 492.93, |
| "jev_latency_n": 200, |
| "jev_latency_concurrency": 1, |
| "hybrid_correct": 2617, |
| "hybrid_accuracy_pct": 88.0, |
| "hybrid_oracle_calls": 1335, |
| "hybrid_api_usd_per_million": 8.48, |
| "jev_api_usd_per_million": 110.86 |
| } |
| ], |
| "notes": [ |
| "Local accuracy is top-1 over every test row before abstention; gate accuracy and coverage are separate.", |
| "Local CPU latency: full text to decision, Apple M4, one CPU thread, 50 warmups, all test rows.", |
| "Cortiq HTTPS latency: same client Mac to deployed server, sequential, no oracle, all test rows.", |
| "Jev latency includes network; BANKING77 uses four workers, CLINC150 and MASSIVE serial samples of 200.", |
| "Different hardware and concurrency: this is not a hardware-normalized throughput comparison.", |
| "Hybrid costs: served cache/learning run with stored DeepSeek answers replayed and 24 live MASSIVE calls; not a timing run.", |
| "Per-million fees extrapolate recorded costs, excluding hardware, electricity and operations.", |
| "Reused public benchmarks, unequal supervision: CMF train+dev vs two examples per label for Jev.", |
| "Current model uses BAAI bge-small-en-v1.5 compressed by Cortiq NVG, not the separate Embryo experiment." |
| ], |
| "sources": [ |
| { |
| "path_in_project": "artifacts/decision-v4-20260926/gates-release.json", |
| "sha256": "47d3039559e0f87dee0c6dae9adc3bbd63acc52b05d4859f2de82cf7a0a9ba03" |
| }, |
| { |
| "path_in_project": "reports/decision-v4-20260926/live/live.json", |
| "sha256": "c8769af100e3138291675ab3eed20caa3ecaa780b6b81f4cecbde96dc29f87f5" |
| }, |
| { |
| "path_in_project": "reports/decision-v4-20260926/article/stored.json", |
| "sha256": "4d6918a2d98f16c8b4cdda7759fef98e6343f091fa7723e8ea938d9fb696db47" |
| }, |
| { |
| "path_in_project": "reports/decision-v4-20260926/eval/stream-final.json", |
| "sha256": "0bb19de37cee315a5a5ad4563abcf2944a80c49ef05984554a4fc31012702f50" |
| } |
| ] |
| } |
|
|
