{
  "verification_report_sha256": "9a5d22ec4b447f0710c12e03c69d7ed802158bb94e9a3c6b059731730ced9f30",
  "evaluator_source_revision": "83f142c4290a0f7d9ed545ae8532858c6e4f8145",
  "evaluator_source_path": "libs/cua-s1/evals/metrics.py",
  "evaluator_sha256": "49dccdeb4e2b456187cdc4491a6d78fabac6a45384272f8e3ec802a58872aa6e",
  "adapter_sha256": "c78e45caa86fb59f9a56ef18cc6a35020ea6e281a6a23d23618f56da85a0d95e",
  "scope": "Offline action/target classification, not GUI execution or task completion",
  "normalization": "Stable row IDs; fill target is entity option index; upstream treats skip as abstention",
  "results": {
    "ALL": {
      "examples": 196,
      "accuracy": 1.0,
      "coverage": 0.23469387755102042,
      "abstention_rate": 0.7653061224489796,
      "selective_accuracy": 1.0,
      "wrong_action_rate": 0.0,
      "wrong_target_rate": 0.0,
      "unsafe_action_rate": 0.0,
      "counts": {
        "total": 196,
        "correct": 196,
        "abstained": 150,
        "acted": 46,
        "acted_correct": 46
      },
      "per_action": {
        "abstain": {
          "accuracy": 1.0,
          "abstention_rate": 1.0,
          "examples": 150
        },
        "check": {
          "accuracy": 1.0,
          "abstention_rate": 0.0,
          "examples": 4
        },
        "click": {
          "accuracy": 1.0,
          "abstention_rate": 0.0,
          "examples": 6
        },
        "fill": {
          "accuracy": 1.0,
          "abstention_rate": 0.0,
          "examples": 36
        }
      }
    },
    "CPU_AND_NE": {
      "examples": 196,
      "accuracy": 1.0,
      "coverage": 0.23469387755102042,
      "abstention_rate": 0.7653061224489796,
      "selective_accuracy": 1.0,
      "wrong_action_rate": 0.0,
      "wrong_target_rate": 0.0,
      "unsafe_action_rate": 0.0,
      "counts": {
        "total": 196,
        "correct": 196,
        "abstained": 150,
        "acted": 46,
        "acted_correct": 46
      },
      "per_action": {
        "abstain": {
          "accuracy": 1.0,
          "abstention_rate": 1.0,
          "examples": 150
        },
        "check": {
          "accuracy": 1.0,
          "abstention_rate": 0.0,
          "examples": 4
        },
        "click": {
          "accuracy": 1.0,
          "abstention_rate": 0.0,
          "examples": 6
        },
        "fill": {
          "accuracy": 1.0,
          "abstention_rate": 0.0,
          "examples": 36
        }
      }
    },
    "upstream_pytorch": {
      "examples": 196,
      "accuracy": 1.0,
      "coverage": 0.23469387755102042,
      "abstention_rate": 0.7653061224489796,
      "selective_accuracy": 1.0,
      "wrong_action_rate": 0.0,
      "wrong_target_rate": 0.0,
      "unsafe_action_rate": 0.0,
      "counts": {
        "total": 196,
        "correct": 196,
        "abstained": 150,
        "acted": 46,
        "acted_correct": 46
      },
      "per_action": {
        "abstain": {
          "accuracy": 1.0,
          "abstention_rate": 1.0,
          "examples": 150
        },
        "check": {
          "accuracy": 1.0,
          "abstention_rate": 0.0,
          "examples": 4
        },
        "click": {
          "accuracy": 1.0,
          "abstention_rate": 0.0,
          "examples": 6
        },
        "fill": {
          "accuracy": 1.0,
          "abstention_rate": 0.0,
          "examples": 36
        }
      }
    }
  }
}