{
  "dataset": "CLINC150",
  "source": "https://github.com/clinc/oos-eval",
  "task": "intent",
  "n_labels": 150,
  "label_wording": "raw intent strings from data_full.json (frozen)",
  "splits": {
    "train": 15000,
    "val": 3000,
    "test": 4500,
    "oos_train": 100,
    "oos_val": 100,
    "oos_test": 1000
  },
  "protocol": "In-scope classification uses all 150 intents. Abstention threshold chosen on val + oos_val only; test + oos_test held out.",
  "onesystem_version": "0.3.0",
  "device": "mps",
  "base_checkpoint": {
    "source": "models/onesystem",
    "max_text_len": 8192,
    "temperature": 2.5000000000000004,
    "encoder_name": "Alibaba-NLP/gte-base-en-v1.5"
  },
  "current_checkpoint": {
    "in_scope_forced": {
      "n": 4500,
      "n_labels": 150,
      "accuracy": 0.7296,
      "macro_f1": 0.7069,
      "ece": 0.4501,
      "nll": 1.8931,
      "coverage": 1.0,
      "accepted_accuracy": 0.7296,
      "abstain_rate": 0.0,
      "min_confidence": null,
      "latency": {
        "n": 50,
        "p50_ms": 10.12,
        "p95_ms": 11.45,
        "mean_ms": 10.18
      },
      "seconds": 9.1
    },
    "threshold_selection": {
      "chosen": 0.1,
      "max_in_scope_rejection": 0.25,
      "eligible_count": 1,
      "validation_grid_summary": [
        {
          "min_confidence": 0.1,
          "oos_recall": 0.95,
          "in_scope_rejection_rate": 0.2403,
          "coverage": 0.7597,
          "accepted_accuracy": 0.8262
        },
        {
          "min_confidence": 0.15,
          "oos_recall": 0.98,
          "in_scope_rejection_rate": 0.3493,
          "coverage": 0.6507,
          "accepted_accuracy": 0.856
        },
        {
          "min_confidence": 0.2,
          "oos_recall": 0.98,
          "in_scope_rejection_rate": 0.4457,
          "coverage": 0.5543,
          "accepted_accuracy": 0.8851
        },
        {
          "min_confidence": 0.25,
          "oos_recall": 0.99,
          "in_scope_rejection_rate": 0.5297,
          "coverage": 0.4703,
          "accepted_accuracy": 0.9121
        },
        {
          "min_confidence": 0.3,
          "oos_recall": 0.99,
          "in_scope_rejection_rate": 0.6037,
          "coverage": 0.3963,
          "accepted_accuracy": 0.931
        },
        {
          "min_confidence": 0.35,
          "oos_recall": 0.99,
          "in_scope_rejection_rate": 0.6687,
          "coverage": 0.3313,
          "accepted_accuracy": 0.9386
        },
        {
          "min_confidence": 0.4,
          "oos_recall": 0.99,
          "in_scope_rejection_rate": 0.724,
          "coverage": 0.276,
          "accepted_accuracy": 0.9577
        },
        {
          "min_confidence": 0.45,
          "oos_recall": 0.99,
          "in_scope_rejection_rate": 0.7753,
          "coverage": 0.2247,
          "accepted_accuracy": 0.9644
        },
        {
          "min_confidence": 0.5,
          "oos_recall": 0.99,
          "in_scope_rejection_rate": 0.8147,
          "coverage": 0.1853,
          "accepted_accuracy": 0.9748
        },
        {
          "min_confidence": 0.55,
          "oos_recall": 1.0,
          "in_scope_rejection_rate": 0.8493,
          "coverage": 0.1507,
          "accepted_accuracy": 0.9801
        },
        {
          "min_confidence": 0.6,
          "oos_recall": 1.0,
          "in_scope_rejection_rate": 0.889,
          "coverage": 0.111,
          "accepted_accuracy": 0.991
        },
        {
          "min_confidence": 0.65,
          "oos_recall": 1.0,
          "in_scope_rejection_rate": 0.921,
          "coverage": 0.079,
          "accepted_accuracy": 1.0
        },
        {
          "min_confidence": 0.7,
          "oos_recall": 1.0,
          "in_scope_rejection_rate": 0.95,
          "coverage": 0.05,
          "accepted_accuracy": 1.0
        },
        {
          "min_confidence": 0.75,
          "oos_recall": 1.0,
          "in_scope_rejection_rate": 0.9707,
          "coverage": 0.0293,
          "accepted_accuracy": 1.0
        },
        {
          "min_confidence": 0.8,
          "oos_recall": 1.0,
          "in_scope_rejection_rate": 0.99,
          "coverage": 0.01,
          "accepted_accuracy": 1.0
        },
        {
          "min_confidence": 0.85,
          "oos_recall": 1.0,
          "in_scope_rejection_rate": 0.9977,
          "coverage": 0.0023,
          "accepted_accuracy": 1.0
        },
        {
          "min_confidence": 0.9,
          "oos_recall": 1.0,
          "in_scope_rejection_rate": 1.0,
          "coverage": 0.0,
          "accepted_accuracy": null
        }
      ]
    },
    "test_with_abstention": {
      "min_confidence": 0.1,
      "oos_n": 1000,
      "oos_recall": 0.89,
      "in_scope_n": 4500,
      "in_scope_rejection_rate": 0.2518,
      "coverage": 0.7482,
      "accepted_accuracy": 0.8298
    },
    "in_scope_with_threshold": {
      "coverage": 0.7482,
      "accepted_accuracy": 0.8298,
      "abstain_rate": 0.2518
    }
  },
  "finetune_split": {
    "train_rows": 13500,
    "calib_rows": 1500
  },
  "domain_adapted": {
    "in_scope_forced": {
      "n": 4500,
      "n_labels": 150,
      "accuracy": 0.9758,
      "macro_f1": 0.9757,
      "ece": 0.0027,
      "nll": 0.1244,
      "coverage": 1.0,
      "accepted_accuracy": 0.9758,
      "abstain_rate": 0.0,
      "min_confidence": null,
      "latency": {
        "n": 50,
        "p50_ms": 10.86,
        "p95_ms": 15.62,
        "mean_ms": 11.72
      },
      "seconds": 10.3
    },
    "threshold_selection": {
      "chosen": 0.9,
      "max_in_scope_rejection": 0.25,
      "eligible_count": 16,
      "validation_grid_summary": [
        {
          "min_confidence": 0.1,
          "oos_recall": 0.09,
          "in_scope_rejection_rate": 0.0,
          "coverage": 1.0,
          "accepted_accuracy": 0.9827
        },
        {
          "min_confidence": 0.15,
          "oos_recall": 0.26,
          "in_scope_rejection_rate": 0.0013,
          "coverage": 0.9987,
          "accepted_accuracy": 0.9836
        },
        {
          "min_confidence": 0.2,
          "oos_recall": 0.34,
          "in_scope_rejection_rate": 0.0017,
          "coverage": 0.9983,
          "accepted_accuracy": 0.984
        },
        {
          "min_confidence": 0.25,
          "oos_recall": 0.48,
          "in_scope_rejection_rate": 0.003,
          "coverage": 0.997,
          "accepted_accuracy": 0.985
        },
        {
          "min_confidence": 0.3,
          "oos_recall": 0.56,
          "in_scope_rejection_rate": 0.004,
          "coverage": 0.996,
          "accepted_accuracy": 0.9859
        },
        {
          "min_confidence": 0.35,
          "oos_recall": 0.63,
          "in_scope_rejection_rate": 0.0047,
          "coverage": 0.9953,
          "accepted_accuracy": 0.9866
        },
        {
          "min_confidence": 0.4,
          "oos_recall": 0.68,
          "in_scope_rejection_rate": 0.0067,
          "coverage": 0.9933,
          "accepted_accuracy": 0.9876
        },
        {
          "min_confidence": 0.45,
          "oos_recall": 0.72,
          "in_scope_rejection_rate": 0.009,
          "coverage": 0.991,
          "accepted_accuracy": 0.9886
        },
        {
          "min_confidence": 0.5,
          "oos_recall": 0.77,
          "in_scope_rejection_rate": 0.0117,
          "coverage": 0.9883,
          "accepted_accuracy": 0.9899
        },
        {
          "min_confidence": 0.55,
          "oos_recall": 0.81,
          "in_scope_rejection_rate": 0.013,
          "coverage": 0.987,
          "accepted_accuracy": 0.9902
        },
        {
          "min_confidence": 0.6,
          "oos_recall": 0.83,
          "in_scope_rejection_rate": 0.015,
          "coverage": 0.985,
          "accepted_accuracy": 0.9902
        },
        {
          "min_confidence": 0.65,
          "oos_recall": 0.84,
          "in_scope_rejection_rate": 0.0177,
          "coverage": 0.9823,
          "accepted_accuracy": 0.9905
        },
        {
          "min_confidence": 0.7,
          "oos_recall": 0.86,
          "in_scope_rejection_rate": 0.0193,
          "coverage": 0.9807,
          "accepted_accuracy": 0.9908
        },
        {
          "min_confidence": 0.75,
          "oos_recall": 0.89,
          "in_scope_rejection_rate": 0.021,
          "coverage": 0.979,
          "accepted_accuracy": 0.9911
        },
        {
          "min_confidence": 0.8,
          "oos_recall": 0.89,
          "in_scope_rejection_rate": 0.0263,
          "coverage": 0.9737,
          "accepted_accuracy": 0.9918
        },
        {
          "min_confidence": 0.85,
          "oos_recall": 0.92,
          "in_scope_rejection_rate": 0.0333,
          "coverage": 0.9667,
          "accepted_accuracy": 0.9921
        },
        {
          "min_confidence": 0.9,
          "oos_recall": 0.94,
          "in_scope_rejection_rate": 0.043,
          "coverage": 0.957,
          "accepted_accuracy": 0.9937
        }
      ]
    },
    "test_with_abstention": {
      "min_confidence": 0.9,
      "oos_n": 1000,
      "oos_recall": 0.917,
      "in_scope_n": 4500,
      "in_scope_rejection_rate": 0.0507,
      "coverage": 0.9493,
      "accepted_accuracy": 0.9918
    },
    "in_scope_with_threshold": {
      "coverage": 0.9493,
      "accepted_accuracy": 0.9918,
      "abstain_rate": 0.0507
    },
    "temperature": 1.7,
    "calib_nll": 0.099,
    "train_run": {
      "steps": 5064,
      "history": [
        {
          "epoch": 1,
          "loss": 0.2156
        },
        {
          "epoch": 2,
          "loss": 0.0434
        },
        {
          "epoch": 3,
          "loss": 0.0113
        }
      ],
      "seconds": 4653.7
    }
  }
}