{
  "title": "OneSystem vs Laya vs OpenJev",
  "protocol": "Official test splits. Every example sees the full candidate label set. Raw = released/default checkpoints. Fine-tuned = OneSystem domain checkpoints trained only on that dataset's train split; Laya/OpenJev public packages have no equivalent documented domain_adapt path in this harness.",
  "systems": {
    "onesystem": "OneSystem v0.3.0 (Alibaba-NLP/gte-base-en-v1.5 init)",
    "laya": "pip install laya \u2192 convaiinnovations/laya (english)",
    "openjev": "Verdict-open-jev (heman10x/rlcd-modernbert-151m) \u2014 open OpenJev"
  },
  "tracks": {
    "fine_tuned": {
      "banking77": [
        {
          "system": "onesystem-banking77",
          "track": "fine_tuned",
          "n": 3080,
          "n_labels": 77,
          "accuracy": 0.9403,
          "macro_f1": 0.9401,
          "ece": 0.012,
          "nll": 0.2424,
          "error_rate": 0.0,
          "seconds": 89.6,
          "latency": {
            "n": 30,
            "p50_ms": 31.48,
            "p95_ms": 37.58,
            "mean_ms": 29.37
          },
          "dataset": "banking77"
        }
      ],
      "clinc150": [
        {
          "system": "onesystem-clinc150",
          "track": "fine_tuned",
          "n": 4500,
          "n_labels": 150,
          "accuracy": 0.9758,
          "macro_f1": 0.9757,
          "ece": 0.0027,
          "nll": 0.1244,
          "error_rate": 0.0,
          "seconds": 131.0,
          "latency": {
            "n": 30,
            "p50_ms": 30.68,
            "p95_ms": 36.48,
            "mean_ms": 30.02
          },
          "dataset": "clinc150"
        }
      ],
      "hwu64": [
        {
          "system": "onesystem-hwu64",
          "track": "fine_tuned",
          "n": 1076,
          "n_labels": 64,
          "accuracy": 0.9368,
          "macro_f1": 0.9338,
          "ece": 0.0201,
          "nll": 0.2745,
          "error_rate": 0.0,
          "seconds": 29.7,
          "latency": {
            "n": 30,
            "p50_ms": 21.7,
            "p95_ms": 24.76,
            "mean_ms": 20.43
          },
          "dataset": "hwu64"
        }
      ]
    },
    "raw": {
      "banking77": [
        {
          "system": "laya",
          "track": "raw",
          "n": 3080,
          "n_labels": 77,
          "accuracy": 0.3802,
          "macro_f1": 0.3326,
          "ece": 0.5279,
          "nll": 9.5297,
          "error_rate": 0.0,
          "seconds": 401.2,
          "latency": {
            "n": 30,
            "p50_ms": 101.72,
            "p95_ms": 107.09,
            "mean_ms": 102.46
          },
          "dataset": "banking77"
        },
        {
          "system": "onesystem",
          "track": "raw",
          "n": 3080,
          "n_labels": 77,
          "accuracy": 0.6334,
          "macro_f1": 0.626,
          "ece": 0.3307,
          "nll": 1.8456,
          "error_rate": 0.0,
          "seconds": 91.4,
          "latency": {
            "n": 30,
            "p50_ms": 29.31,
            "p95_ms": 35.36,
            "mean_ms": 27.78
          },
          "dataset": "banking77"
        },
        {
          "system": "openjev",
          "track": "raw",
          "n": 3080,
          "n_labels": 77,
          "accuracy": 0.5325,
          "macro_f1": 0.5506,
          "ece": 0.2738,
          "nll": 9.764,
          "error_rate": 0.0,
          "seconds": 820.6,
          "latency": {
            "n": 30,
            "p50_ms": 288.56,
            "p95_ms": 308.62,
            "mean_ms": 293.57
          },
          "dataset": "banking77"
        }
      ],
      "hwu64": [
        {
          "system": "laya",
          "track": "raw",
          "n": 1076,
          "n_labels": 64,
          "accuracy": 0.3522,
          "macro_f1": 0.3048,
          "ece": 0.306,
          "nll": 3.6672,
          "error_rate": 0.0,
          "seconds": 92.0,
          "latency": {
            "n": 30,
            "p50_ms": 85.03,
            "p95_ms": 87.65,
            "mean_ms": 85.04
          },
          "dataset": "hwu64"
        },
        {
          "system": "onesystem",
          "track": "raw",
          "n": 1076,
          "n_labels": 64,
          "accuracy": 0.645,
          "macro_f1": 0.5976,
          "ece": 0.4091,
          "nll": 2.0467,
          "error_rate": 0.0,
          "seconds": 30.4,
          "latency": {
            "n": 30,
            "p50_ms": 29.47,
            "p95_ms": 35.32,
            "mean_ms": 27.51
          },
          "dataset": "hwu64"
        },
        {
          "system": "openjev",
          "track": "raw",
          "n": 1076,
          "n_labels": 64,
          "accuracy": 0.2965,
          "macro_f1": 0.3479,
          "ece": 0.1089,
          "nll": 12.9571,
          "error_rate": 0.0,
          "seconds": 188.2,
          "latency": {
            "n": 30,
            "p50_ms": 211.23,
            "p95_ms": 223.9,
            "mean_ms": 213.77
          },
          "dataset": "hwu64"
        }
      ],
      "clinc150": [
        {
          "system": "laya",
          "track": "raw",
          "n": 4500,
          "n_labels": 150,
          "accuracy": 0.5324,
          "macro_f1": 0.5008,
          "ece": 0.3395,
          "nll": 4.3061,
          "error_rate": 0.0,
          "seconds": 960.2,
          "latency": {
            "n": 30,
            "p50_ms": 210.86,
            "p95_ms": 224.13,
            "mean_ms": 211.72
          },
          "dataset": "clinc150"
        },
        {
          "system": "onesystem",
          "track": "raw",
          "n": 4500,
          "n_labels": 150,
          "accuracy": 0.7296,
          "macro_f1": 0.7069,
          "ece": 0.4501,
          "nll": 1.8931,
          "error_rate": 0.0,
          "seconds": 132.5,
          "latency": {
            "n": 30,
            "p50_ms": 30.47,
            "p95_ms": 40.33,
            "mean_ms": 29.63
          },
          "dataset": "clinc150"
        },
        {
          "system": "openjev",
          "track": "raw",
          "n": 4500,
          "n_labels": 150,
          "accuracy": 0.5522,
          "macro_f1": 0.5587,
          "ece": 0.0969,
          "nll": 10.3148,
          "error_rate": 0.0,
          "seconds": 2059.0,
          "latency": {
            "n": 30,
            "p50_ms": 473.7,
            "p95_ms": 498.4,
            "mean_ms": 477.39
          },
          "dataset": "clinc150"
        }
      ]
    }
  },
  "notes": [
    "PyPI package `openjev==0.0.1` is an empty stub; this board uses Verdict-open-jev.",
    "TypeSafe hosted Jev is not evaluated (API key / waitlist).",
    "Holdout fast-decisions scores are a different evaluation and are not mixed in here."
  ]
}