{
  "algorithm_version": "m3_intent_v1",
  "model_kind": "JointBERT: intent + BIO slots + multi-label variables on a shared encoder",
  "base_model": "google/muril-base-cased",
  "dataset_kind": "d4_multilingual_templated_queries_with_exact_slot_spans",
  "dataset_path": "/data/d4_queries.parquet",
  "dataset_sha256": "cb2ee6c4e1d8d9857975d3362957d90cf6ba8f8e242a4441af264c17645207fc",
  "split": "held-out template families {sow,heat,storm} AND 20% held-out districts",
  "n_train": 2340,
  "n_val": 412,
  "n_test": 1275,
  "epochs": 8,
  "batch_size": 32,
  "lr": 3e-05,
  "seed": 42,
  "trained_at": "2026-09-04T08:43:34.868234Z",
  "val": {
    "n": 412,
    "intent_accuracy": 1.0,
    "intent_macro_f1": 1.0,
    "slot_f1": 0.9840571742715778,
    "slot_precision": 0.9955506117908788,
    "slot_recall": 0.9728260869565217,
    "variable_micro_f1": 0.34218436873747493,
    "variable_macro_f1": 0.20806501485889486,
    "variable_exact_set_match": 0.0
  },
  "test_heldout": {
    "n": 1275,
    "intent_accuracy": 0.836078431372549,
    "intent_macro_f1": 0.7459041972533924,
    "slot_f1": 0.9878318584070797,
    "slot_precision": 0.9900221729490022,
    "slot_recall": 0.9856512141280354,
    "variable_micro_f1": 0.1703968716199351,
    "variable_macro_f1": 0.12022848845265091,
    "variable_exact_set_match": 0.0
  },
  "per_language_test": {
    "as": {
      "n": 91,
      "intent_accuracy": 0.8241758241758241,
      "intent_macro_f1": 0.7046130952380951,
      "slot_f1": 0.9819121447028423,
      "slot_precision": 0.9844559585492227,
      "slot_recall": 0.979381443298969,
      "variable_micro_f1": 0.16258351893095768,
      "variable_macro_f1": 0.11573187976125571,
      "variable_exact_set_match": 0.0
    },
    "bn": {
      "n": 91,
      "intent_accuracy": 0.8241758241758241,
      "intent_macro_f1": 0.7485653235653236,
      "slot_f1": 0.9922480620155039,
      "slot_precision": 0.9948186528497409,
      "slot_recall": 0.9896907216494846,
      "variable_micro_f1": 0.16252821670428894,
      "variable_macro_f1": 0.11745691007894474,
      "variable_exact_set_match": 0.0
    },
    "bn_latn": {
      "n": 91,
      "intent_accuracy": 0.8241758241758241,
      "intent_macro_f1": 0.7774305555555556,
      "slot_f1": 0.96875,
      "slot_precision": 0.9789473684210527,
      "slot_recall": 0.9587628865979382,
      "variable_micro_f1": 0.18472906403940886,
      "variable_macro_f1": 0.13375335824197646,
      "variable_exact_set_match": 0.0
    },
    "en": {
      "n": 94,
      "intent_accuracy": 0.8297872340425532,
      "intent_macro_f1": 0.7477470930232558,
      "slot_f1": 0.982367758186398,
      "slot_precision": 0.9898477157360406,
      "slot_recall": 0.975,
      "variable_micro_f1": 0.16666666666666666,
      "variable_macro_f1": 0.10588481102765866,
      "variable_exact_set_match": 0.0
    },
    "gu": {
      "n": 91,
      "intent_accuracy": 0.7472527472527473,
      "intent_macro_f1": 0.7242408999451253,
      "slot_f1": 0.9948453608247423,
      "slot_precision": 0.9948453608247423,
      "slot_recall": 0.9948453608247423,
      "variable_micro_f1": 0.16497175141242937,
      "variable_macro_f1": 0.119454266228756,
      "variable_exact_set_match": 0.0
    },
    "hi": {
      "n": 91,
      "intent_accuracy": 0.8241758241758241,
      "intent_macro_f1": 0.747470238095238,
      "slot_f1": 0.987012987012987,
      "slot_precision": 0.9947643979057592,
      "slot_recall": 0.979381443298969,
      "variable_micro_f1": 0.17478152309612985,
      "variable_macro_f1": 0.11732083168612839,
      "variable_exact_set_match": 0.0
    },
    "hi_latn": {
      "n": 91,
      "intent_accuracy": 0.8351648351648352,
      "intent_macro_f1": 0.7508169934640523,
      "slot_f1": 0.9690721649484536,
      "slot_precision": 0.9690721649484536,
      "slot_recall": 0.9690721649484536,
      "variable_micro_f1": 0.17262638717632553,
      "variable_macro_f1": 0.11856133517082122,
      "variable_exact_set_match": 0.0
    },
    "kn": {
      "n": 91,
      "intent_accuracy": 0.8901098901098901,
      "intent_macro_f1": 0.7679118773946361,
      "slot_f1": 0.9948453608247423,
      "slot_precision": 0.9948453608247423,
      "slot_recall": 0.9948453608247423,
      "variable_micro_f1": 0.17067833698030635,
      "variable_macro_f1": 0.12361479023030504,
      "variable_exact_set_match": 0.0
    },
    "ml": {
      "n": 89,
      "intent_accuracy": 0.8426966292134831,
      "intent_macro_f1": 0.753756884343037,
      "slot_f1": 1.0,
      "slot_precision": 1.0,
      "slot_recall": 1.0,
      "variable_micro_f1": 0.16666666666666666,
      "variable_macro_f1": 0.12288051655026812,
      "variable_exact_set_match": 0.0
    },
    "mr": {
      "n": 91,
      "intent_accuracy": 0.8241758241758241,
      "intent_macro_f1": 0.747470238095238,
      "slot_f1": 0.9948453608247423,
      "slot_precision": 0.9948453608247423,
      "slot_recall": 0.9948453608247423,
      "variable_micro_f1": 0.170316301703163,
      "variable_macro_f1": 0.11399250334403123,
      "variable_exact_set_match": 0.0
    },
    "or": {
      "n": 91,
      "intent_accuracy": 0.8791208791208791,
      "intent_macro_f1": 0.7644305846505428,
      "slot_f1": 0.9922879177377892,
      "slot_precision": 0.9897435897435898,
      "slot_recall": 0.9948453608247423,
      "variable_micro_f1": 0.1736111111111111,
      "variable_macro_f1": 0.122160447935823,
      "variable_exact_set_match": 0.0
    },
    "pa": {
      "n": 91,
      "intent_accuracy": 0.8241758241758241,
      "intent_macro_f1": 0.747470238095238,
      "slot_f1": 0.9819121447028423,
      "slot_precision": 0.9844559585492227,
      "slot_recall": 0.979381443298969,
      "variable_micro_f1": 0.17582417582417584,
      "variable_macro_f1": 0.13613372093804568,
      "variable_exact_set_match": 0.0
    },
    "ta": {
      "n": 91,
      "intent_accuracy": 0.9120879120879121,
      "intent_macro_f1": 0.775,
      "slot_f1": 0.9948453608247423,
      "slot_precision": 0.9948453608247423,
      "slot_recall": 0.9948453608247423,
      "variable_micro_f1": 0.17391304347826086,
      "variable_macro_f1": 0.12383695029450821,
      "variable_exact_set_match": 0.0
    },
    "te": {
      "n": 91,
      "intent_accuracy": 0.8241758241758241,
      "intent_macro_f1": 0.747470238095238,
      "slot_f1": 0.9948453608247423,
      "slot_precision": 0.9948453608247423,
      "slot_recall": 0.9948453608247423,
      "variable_micro_f1": 0.16799091940976163,
      "variable_macro_f1": 0.11866599643861585,
      "variable_exact_set_match": 0.0
    }
  },
  "baselines": {
    "majority_class_test": {
      "intent_accuracy": 0.504313725490196,
      "intent_macro_f1": 0.08381126173096975
    },
    "rule_based_retrieval_planner_test": {
      "intent_accuracy": 0.5129411764705882,
      "intent_macro_f1": 0.16641001977410239
    }
  }
}