eval/model_eval_summary.json

{
 "val": {
  "n": 10413,
  "exact": 0.991,
  "action": 0.9929,
  "top_confusions": [
   [
    [
     "get_hostname",
     "unknown"
    ],
    9
   ],
   [
    [
     "rename_file",
     "hash_file"
    ],
    6
   ],
   [
    [
     "unknown",
     "help"
    ],
    5
   ],
   [
    [
     "unknown",
     "type_text"
    ],
    4
   ],
   [
    [
     "unknown",
     "connect_wifi"
    ],
    3
   ],
   [
    [
     "unknown",
     "hibernate"
    ],
    2
   ],
   [
    [
     "set_alarm",
     "set_alarm"
    ],
    2
   ],
   [
    [
     "get_cpu_clock",
     "get_time"
    ],
    2
   ],
   [
    [
     "shutdown",
     "shutdown"
    ],
    2
   ],
   [
    [
     "youtube_search",
     "youtube_search"
    ],
    2
   ],
   [
    [
     "search_files",
     "search_files"
    ],
    2
   ],
   [
    [
     "type_text",
     "type_text"
    ],
    2
   ]
  ],
  "secs": 21
 },
 "test": {
  "n": 10618,
  "exact": 0.9837,
  "action": 0.9852,
  "top_confusions": [
   [
    [
     "unknown",
     "clarify"
    ],
    19
   ],
   [
    [
     "unknown",
     "get_reboot_history"
    ],
    18
   ],
   [
    [
     "format_drive",
     "get_sdcard_info"
    ],
    15
   ],
   [
    [
     "unknown",
     "web_search"
    ],
    10
   ],
   [
    [
     "unknown",
     "cancel_shutdown"
    ],
    10
   ],
   [
    [
     "discover_devices",
     "scan_lan"
    ],
    9
   ],
   [
    [
     "board_to_bcm",
     "bcm_to_board"
    ],
    8
   ],
   [
    [
     "unknown",
     "shutdown"
    ],
    7
   ],
   [
    [
     "search_in_files",
     "search_files"
    ],
    7
   ],
   [
    [
     "close_app",
     "uninstall_app"
    ],
    6
   ],
   [
    [
     "service_start",
     "service_enable"
    ],
    5
   ],
   [
    [
     "unknown",
     "undo"
    ],
    4
   ]
  ],
  "secs": 19
 },
 "test_strict": {
  "n": 6611,
  "exact": 0.9753,
  "action": 0.977,
  "top_confusions": [],
  "secs": 0
 },
 "curated": {
  "n": 371,
  "exact": 0.9946,
  "action": 0.9973,
  "top_confusions": [
   [
    [
     "gpio_on",
     "gpio_on"
    ],
    1
   ],
   [
    [
     "discover_devices",
     "scan_lan"
    ],
    1
   ]
  ],
  "secs": 1
 },
 "curated_strict": {
  "n": 253,
  "exact": 0.9921,
  "action": 0.996,
  "top_confusions": [],
  "secs": 0
 },
 "A_challenge": {
  "n": 1152,
  "exact": 0.974,
  "action": 0.9844,
  "top_confusions": [
   [
    [
     "open_app",
     "open_app"
    ],
    6
   ],
   [
    [
     "read_file",
     "read_file"
    ],
    6
   ],
   [
    [
     "unknown",
     "set_volume"
    ],
    6
   ],
   [
    [
     "unknown",
     "format_drive"
    ],
    6
   ],
   [
    [
     "unknown",
     "cancel_shutdown"
    ],
    4
   ],
   [
    [
     "media_play",
     "media_play_pause"
    ],
    1
   ],
   [
    [
     "unknown",
     "bookmark_page"
    ],
    1
   ]
  ],
  "secs": 1
 },
 "A_practical-dev": {
  "n": 1321,
  "exact": 0.9531,
  "action": 0.9644,
  "top_confusions": [
   [
    [
     "read_file",
     "read_file"
    ],
    8
   ],
   [
    [
     "unknown",
     "format_drive"
    ],
    8
   ],
   [
    [
     "open_app",
     "open_app"
    ],
    6
   ],
   [
    [
     "unknown",
     "set_volume"
    ],
    6
   ],
   [
    [
     "set_brightness",
     "clarify"
    ],
    5
   ],
   [
    [
     "unknown",
     "cancel_shutdown"
    ],
    4
   ],
   [
    [
     "set_volume",
     "clarify"
    ],
    4
   ],
   [
    [
     "unknown",
     "factory_reset"
    ],
    3
   ],
   [
    [
     "clarify",
     "set_brightness"
    ],
    3
   ],
   [
    [
     "clarify",
     "set_volume"
    ],
    2
   ],
   [
    [
     "media_play",
     "media_play_pause"
    ],
    1
   ],
   [
    [
     "unknown",
     "bookmark_page"
    ],
    1
   ]
  ],
  "secs": 1
 },
 "A_practical-holdout": {
  "n": 428,
  "exact": 0.9206,
  "action": 0.9416,
  "top_confusions": [
   [
    [
     "clarify",
     "set_volume"
    ],
    4
   ],
   [
    [
     "ping",
     "ping"
    ],
    3
   ],
   [
    [
     "open_app",
     "open_app"
    ],
    2
   ],
   [
    [
     "list_files",
     "search_in_files"
    ],
    2
   ],
   [
    [
     "list_partitions",
     "unknown"
    ],
    2
   ],
   [
    [
     "read_file",
     "read_file"
    ],
    2
   ],
   [
    [
     "open_app",
     "open_file"
    ],
    1
   ],
   [
    [
     "get_system_info",
     "list_services"
    ],
    1
   ],
   [
    [
     "service_stop",
     "service_start"
    ],
    1
   ],
   [
    [
     "run_script",
     "format_drive"
    ],
    1
   ],
   [
    [
     "run_script",
     "factory_reset"
    ],
    1
   ],
   [
    [
     "set_volume",
     "set_volume"
    ],
    1
   ]
  ],
  "secs": 1
 },
 "B_golden_dev": {
  "n": 383,
  "exact": 0.9504,
  "action": 0.9582,
  "top_confusions": [
   [
    [
     "youtube_search",
     "youtube_search"
    ],
    1
   ],
   [
    [
     "unknown",
     "volume_up"
    ],
    1
   ],
   [
    [
     "open_site",
     "web_search"
    ],
    1
   ],
   [
    [
     "restart",
     "cancel_shutdown"
    ],
    1
   ],
   [
    [
     "unknown",
     "get_temperature"
    ],
    1
   ],
   [
    [
     "unknown",
     "get_kernel_log"
    ],
    1
   ],
   [
    [
     "open_app",
     "open_site"
    ],
    1
   ],
   [
    [
     "unknown",
     "open_file"
    ],
    1
   ],
   [
    [
     "open_app",
     "open_app"
    ],
    1
   ],
   [
    [
     "unknown",
     "set_display_scale"
    ],
    1
   ],
   [
    [
     "unknown",
     "recent_files"
    ],
    1
   ],
   [
    [
     "unknown",
     "lock_screen"
    ],
    1
   ]
  ],
  "secs": 0
 },
 "B_golden_final": {
  "n": 384,
  "exact": 0.974,
  "action": 0.974,
  "top_confusions": [
   [
    [
     "unknown",
     "get_date"
    ],
    2
   ],
   [
    [
     "ping",
     "clarify"
    ],
    1
   ],
   [
    [
     "unknown",
     "network_status"
    ],
    1
   ],
   [
    [
     "unknown",
     "shutdown"
    ],
    1
   ],
   [
    [
     "unknown",
     "get_time"
    ],
    1
   ],
   [
    [
     "unknown",
     "clipboard_history"
    ],
    1
   ],
   [
    [
     "bluetooth_on",
     "bluetooth_connect"
    ],
    1
   ],
   [
    [
     "set_brightness",
     "brightness_up"
    ],
    1
   ],
   [
    [
     "unknown",
     "help"
    ],
    1
   ]
  ],
  "secs": 0
 },
 "B_typo": {
  "n": 3000,
  "exact": 0.9917,
  "action": 0.995,
  "top_confusions": [
   [
    [
     "youtube_search",
     "youtube_search"
    ],
    3
   ],
   [
    [
     "close_app",
     "close_app"
    ],
    2
   ],
   [
    [
     "gpio_on",
     "gpio_on"
    ],
    2
   ],
   [
    [
     "battery_saver_off",
     "battery_saver_on"
    ],
    1
   ],
   [
    [
     "media_pause",
     "paste"
    ],
    1
   ],
   [
    [
     "service_start",
     "service_enable"
    ],
    1
   ],
   [
    [
     "open_site",
     "unknown"
    ],
    1
   ],
   [
    [
     "set_brightness",
     "brightness_down"
    ],
    1
   ],
   [
    [
     "open_app",
     "unknown"
    ],
    1
   ],
   [
    [
     "open_folder",
     "open_file"
    ],
    1
   ],
   [
    [
     "open_settings",
     "open_settings"
    ],
    1
   ],
   [
    [
     "wifi_off",
     "cut"
    ],
    1
   ]
  ],
  "secs": 4
 },
 "wifi_holdout": {
  "n": 461,
  "exact": 1.0,
  "action": 1.0,
  "top_confusions": [],
  "secs": 1
 },
 "gap10_holdout": {
  "n": 1401,
  "exact": 1.0,
  "action": 1.0,
  "top_confusions": [],
  "secs": 2
 },
 "gap11_holdout": {
  "n": 696,
  "exact": 0.9928,
  "action": 1.0,
  "top_confusions": [
   [
    [
     "connect_wifi",
     "connect_wifi"
    ],
    5
   ]
  ],
  "secs": 1
 }
}