{ "models": [ { "name": "Smartwatch LM v0.2", "export_dir": ".", "device": "cpu", "best_val_loss": 0.32432237863540647, "intent_accuracy": 1.0, "intent_parse_rate": 1.0, "clean_output_rate": 1.0, "slot_presence": 0.9615384615384616, "results": [ { "case_id": "none_hi_00", "prompt": "hi", "expected_intent": "NONE", "predicted_intent": "NONE", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Hey there — good to see you .", "display_reply": "Hey there — good to see you ." }, { "case_id": "get_steps_hey_am_i_doing_okay_with_steps_today_or_01", "prompt": "Hey — am I doing okay with steps today or should I try to move more?", "expected_intent": "GET_STEPS", "predicted_intent": "GET_STEPS", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " You' re at of . That leaves to go .", "display_reply": "You' re at of . That leaves to go ." }, { "case_id": "set_step_goal_i_want_to_change_my_default_daily_step_t_02", "prompt": "I want to change my default daily step target to 10,000 steps — can you update it?", "expected_intent": "SET_STEP_GOAL", "predicted_intent": "SET_STEP_GOAL", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Done — your daily step goal is now . I'll track progress on your watch and nudge you if you need a mid - afternoon .", "display_reply": "Done — your daily step goal is now . I'll track progress on your watch and nudge you if you need a mid - afternoon ." }, { "case_id": "get_workout_status_how_long_have_i_been_on_my_run_and_what_03", "prompt": "How long have I been on my run, and what's my heart rate right now?", "expected_intent": "GET_WORKOUT_STATUS", "predicted_intent": "GET_WORKOUT_STATUS", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " You're into your session .", "display_reply": "You're into your session ." }, { "case_id": "get_distance_quick_check_what_s_my_total_distance_tod_04", "prompt": "Quick check — what's my total distance today?", "expected_intent": "GET_DISTANCE", "predicted_intent": "GET_DISTANCE", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " You've logged today .", "display_reply": "You've logged today ." }, { "case_id": "get_battery_quick_question_is_my_battery_high_enough_05", "prompt": "Quick question — is my battery high enough to track a walk? Also, how many steps have I done today?", "expected_intent": "GET_BATTERY", "predicted_intent": "GET_BATTERY", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Battery is at — plenty for a walk .", "display_reply": "Battery is at — plenty for a walk ." }, { "case_id": "get_heart_rate_can_you_check_my_heart_rate_now_i_m_a_bi_06", "prompt": "Can you check my heart rate now? I'm a bit worried.", "expected_intent": "GET_HEART_RATE", "predicted_intent": "GET_HEART_RATE", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Your current heart rate is . That's a bit elevated , but if you feel dizzy , short of breath , or chest pain", "display_reply": "Your current heart rate is . That's a bit elevated , but if you feel dizzy , short of breath , or chest pain" }, { "case_id": "measure_heart_rate_please_take_my_pulse_now_i_feel_a_little_07", "prompt": "Please take my pulse now — I feel a little fluttery.", "expected_intent": "MEASURE_HEART_RATE", "predicted_intent": "MEASURE_HEART_RATE", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 0.0, "raw_reply": " I' m measuring your pulse now . Please hold still for a moment .", "display_reply": "I' m measuring your pulse now . Please hold still for a moment ." }, { "case_id": "get_sleep_hey_how_long_did_i_sleep_last_night_08", "prompt": "Hey, how long did I sleep last night?", "expected_intent": "GET_SLEEP", "predicted_intent": "GET_SLEEP", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " You slept last night — nice work .", "display_reply": "You slept last night — nice work ." }, { "case_id": "log_nap_just_woke_up_from_a_quick_power_nap_can_09", "prompt": "Just woke up from a quick power nap — can you log it?", "expected_intent": "LOG_NAP", "predicted_intent": "LOG_NAP", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it , logging a nap of . You had last night , so this should help a bit .", "display_reply": "Got it , logging a nap of . You had last night , so this should help a bit ." }, { "case_id": "get_calories_hey_how_many_calories_have_i_burned_sinc_10", "prompt": "Hey, how many calories have I burned since midnight?", "expected_intent": "GET_CALORIES", "predicted_intent": "GET_CALORIES", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " You' ve burned so far today , and your goal is — nice work .", "display_reply": "You' ve burned so far today , and your goal is — nice work ." }, { "case_id": "set_calorie_goal_lower_my_daily_active_calorie_burn_targe_11", "prompt": "Lower my daily active calorie burn target to 400, please.", "expected_intent": "SET_CALORIE_GOAL", "predicted_intent": "SET_CALORIE_GOAL", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Done — your daily active calorie goal is now . I' ll keep track and let you know how you' re doing .", "display_reply": "Done — your daily active calorie goal is now . I' ll keep track and let you know how you' re doing ." }, { "case_id": "get_active_minutes_how_many_active_minutes_have_i_done_toda_12", "prompt": "How many active minutes have I done today?", "expected_intent": "GET_ACTIVE_MINUTES", "predicted_intent": "GET_ACTIVE_MINUTES", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " You' ve got active minutes so far , with a goal of .", "display_reply": "You' ve got active minutes so far , with a goal of ." }, { "case_id": "set_active_goal_can_you_raise_my_daily_active_minutes_go_13", "prompt": "Can you raise my daily active minutes goal to 45 minutes?", "expected_intent": "SET_ACTIVE_GOAL", "predicted_intent": "SET_ACTIVE_GOAL", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Done — your daily active minutes goal is now .", "display_reply": "Done — your daily active minutes goal is now ." }, { "case_id": "enable_power_save_battery_s_low_turn_on_power_saving_now_14", "prompt": "Battery's low — turn on power saving now.", "expected_intent": "ENABLE_POWER_SAVE", "predicted_intent": "ENABLE_POWER_SAVE", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Power saving is on . Battery is at , so I'll dim the screen , limit background sync , and reduce sensor use .", "display_reply": "Power saving is on . Battery is at , so I'll dim the screen , limit background sync , and reduce sensor use ." }, { "case_id": "disable_aod_please_turn_off_the_always_on_display_to_15", "prompt": "Please turn off the always-on display to save battery.", "expected_intent": "DISABLE_AOD", "predicted_intent": "DISABLE_AOD", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Done — always - on display is off , and your battery is at .", "display_reply": "Done — always - on display is off , and your battery is at ." }, { "case_id": "set_alarm_i_need_an_alarm_for_tomorrow_morning_to_16", "prompt": "I need an alarm for tomorrow morning to make sure I wake up for my run.", "expected_intent": "SET_ALARM", "predicted_intent": "SET_ALARM", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — I set an alarm for on with the label .", "display_reply": "Got it — I set an alarm for on with the label ." }, { "case_id": "list_alarms_quick_check_what_alarms_are_set_on_my_wa_17", "prompt": "Quick check — what alarms are set on my watch?", "expected_intent": "LIST_ALARMS", "predicted_intent": "LIST_ALARMS", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Here are your active alarms : — ; — ; ", "display_reply": "Here are your active alarms : — ; — ; " }, { "case_id": "delete_alarm_please_delete_my_standard_morning_wake_u_18", "prompt": "Please delete my standard morning wake-up alarm.", "expected_intent": "DELETE_ALARM", "predicted_intent": "DELETE_ALARM", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — I deleted .", "display_reply": "Got it — I deleted ." }, { "case_id": "explain_nudge_hey_why_did_my_watch_just_vibrate_19", "prompt": "hey, why did my watch just vibrate?", "expected_intent": "EXPLAIN_NUDGE", "predicted_intent": "EXPLAIN_NUDGE", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " It looks like a sedentary move nudge for — you' ve been still for , so it' s reminding you to get up and", "display_reply": "It looks like a sedentary move nudge for — you' ve been still for , so it' s reminding you to get up and" }, { "case_id": "mute_reminders_i_m_getting_hourly_move_reminders_mute_t_20", "prompt": "I'm getting hourly move reminders — mute them entirely, please.", "expected_intent": "MUTE_REMINDERS", "predicted_intent": "MUTE_REMINDERS", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — I' ve muted the hourly move reminders on your watch .", "display_reply": "Got it — I' ve muted the hourly move reminders on your watch ." }, { "case_id": "snooze_alarm_ugh_it_s_buzzing_snooze_this_alarm_for_a_21", "prompt": "Ugh, it's buzzing — snooze this alarm for a few minutes.", "expected_intent": "SNOOZE_ALARM", "predicted_intent": "SNOOZE_ALARM", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — snoozing set for for . I'll buzz you again then .", "display_reply": "Got it — snoozing set for for . I'll buzz you again then ." }, { "case_id": "set_reminder_i_keep_forgetting_to_drink_water_can_you_22", "prompt": "I keep forgetting to drink water — can you set a repeating reminder every two hours to drink water, please?", "expected_intent": "SET_REMINDER", "predicted_intent": "SET_REMINDER", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — I' ll set a repeating reminder for every .", "display_reply": "Got it — I' ll set a repeating reminder for every ." }, { "case_id": "start_timer_start_a_twenty_minute_timer_23", "prompt": "Start a twenty-minute timer.", "expected_intent": "START_TIMER", "predicted_intent": "START_TIMER", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — starting a timer for .", "display_reply": "Got it — starting a timer for ." }, { "case_id": "get_timer_remaining_how_much_time_is_left_on_the_cooking_tim_24", "prompt": "How much time is left on the cooking timer? I'm juggling a few things here.", "expected_intent": "GET_TIMER_REMAINING", "predicted_intent": "GET_TIMER_REMAINING", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " The timer has remaining out of the .", "display_reply": "The timer has remaining out of the ." }, { "case_id": "pause_timer_pause_the_countdown_now_freeze_it_25", "prompt": "Pause the countdown now — freeze it.", "expected_intent": "PAUSE_TIMER", "predicted_intent": "PAUSE_TIMER", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Paused your timer — remaining of the . I'll keep it frozen until you say resume .", "display_reply": "Paused your timer — remaining of the . I'll keep it frozen until you say resume ." }, { "case_id": "start_stopwatch_start_the_stopwatch_app_and_begin_timing_26", "prompt": "Start the stopwatch app and begin timing.", "expected_intent": "START_STOPWATCH", "predicted_intent": "START_STOPWATCH", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Stopwatch started — I'll record time on your wrist . Tap to lap or say \" stop \" to stop .", "display_reply": "Stopwatch started — I'll record time on your wrist . Tap to lap or say \" stop \" to stop ." }, { "case_id": "lap_stopwatch_mark_a_lap_split_27", "prompt": "Mark a lap split.", "expected_intent": "LAP_STOPWATCH", "predicted_intent": "LAP_STOPWATCH", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — lap recorded at , with the run at .", "display_reply": "Got it — lap recorded at , with the run at ." }, { "case_id": "reset_stopwatch_hey_reset_my_stopwatch_and_clear_its_lap_28", "prompt": "Hey, reset my stopwatch and clear its laps please.", "expected_intent": "RESET_STOPWATCH", "predicted_intent": "RESET_STOPWATCH", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Stopwatch reset to , and the last lap is cleared from .", "display_reply": "Stopwatch reset to , and the last lap is cleared from ." }, { "case_id": "cancel_timer_stop_and_clear_my_running_timer_please_29", "prompt": "Stop and clear my running timer, please.", "expected_intent": "CANCEL_TIMER", "predicted_intent": "CANCEL_TIMER", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 0.5, "raw_reply": " Got it — I canceled the timer with left and cleared it out .", "display_reply": "Got it — I canceled the timer with left and cleared it out ." }, { "case_id": "start_workout_i_m_heading_out_for_an_outdoor_walk_star_30", "prompt": "I'm heading out for an outdoor walk — start tracking, please.", "expected_intent": "START_WORKOUT", "predicted_intent": "START_WORKOUT", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — starting now . Enjoy the walk .", "display_reply": "Got it — starting now . Enjoy the walk ." }, { "case_id": "stop_workout_i_m_done_with_my_run_stop_tracking_and_s_31", "prompt": "I'm done with my run — stop tracking and save the session.", "expected_intent": "STOP_WORKOUT", "predicted_intent": "STOP_WORKOUT", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it , I stopped your and saved the session at . Nice work today .", "display_reply": "Got it , I stopped your and saved the session at . Nice work today ." }, { "case_id": "pause_workout_traffic_light_pause_my_workout_while_i_w_32", "prompt": "Traffic light—pause my workout while I wait?", "expected_intent": "PAUSE_WORKOUT", "predicted_intent": "PAUSE_WORKOUT", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Pausing your workout now . You've been active for . Current state : . I'll keep it paused until you resume .", "display_reply": "Pausing your workout now . You've been active for . Current state : . I'll keep it paused until you resume ." }, { "case_id": "resume_workout_hey_resume_my_cycling_workout_from_earli_33", "prompt": "hey, resume my cycling workout from earlier", "expected_intent": "RESUME_WORKOUT", "predicted_intent": "RESUME_WORKOUT", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Resuming your workout . You' re back in , with already logged .", "display_reply": "Resuming your workout . You' re back in , with already logged ." }, { "case_id": "discard_workout_delete_the_current_run_and_erase_its_dat_34", "prompt": "Delete the current run and erase its data.", "expected_intent": "DISCARD_WORKOUT", "predicted_intent": "DISCARD_WORKOUT", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — I' ll discard the while it' s .", "display_reply": "Got it — I' ll discard the while it' s ." }, { "case_id": "get_workout_summary_just_finished_a_walk_show_the_final_summ_35", "prompt": "Just finished a walk — show the final summary screen.", "expected_intent": "GET_WORKOUT_SUMMARY", "predicted_intent": "GET_WORKOUT_SUMMARY", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Here' s your summary : elapsed and covered today .", "display_reply": "Here' s your summary : elapsed and covered today ." }, { "case_id": "get_steps_how_many_steps_have_i_taken_today_36", "prompt": "How many steps have I taken today?", "expected_intent": "GET_STEPS", "predicted_intent": "GET_STEPS", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " You' re at of so far .", "display_reply": "You' re at of so far ." }, { "case_id": "start_timer_start_a_10_minute_timer_37", "prompt": "Start a 10 minute timer", "expected_intent": "START_TIMER", "predicted_intent": "START_TIMER", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " Got it — starting a timer for .", "display_reply": "Got it — starting a timer for ." }, { "case_id": "none_hey_38", "prompt": "hey", "expected_intent": "NONE", "predicted_intent": "NONE", "intent_correct": true, "has_intent_tag": true, "clean_output": true, "slot_presence": 1.0, "raw_reply": " hi there — I'm listening .", "display_reply": "hi there — I'm listening ." } ], "elapsed_sec": 31.05121770000551, "per_intent": { "CANCEL_TIMER": { "correct": 1, "total": 1, "accuracy": 1.0 }, "DELETE_ALARM": { "correct": 1, "total": 1, "accuracy": 1.0 }, "DISABLE_AOD": { "correct": 1, "total": 1, "accuracy": 1.0 }, "DISCARD_WORKOUT": { "correct": 1, "total": 1, "accuracy": 1.0 }, "ENABLE_POWER_SAVE": { "correct": 1, "total": 1, "accuracy": 1.0 }, "EXPLAIN_NUDGE": { "correct": 1, "total": 1, "accuracy": 1.0 }, "GET_ACTIVE_MINUTES": { "correct": 1, "total": 1, "accuracy": 1.0 }, "GET_BATTERY": { "correct": 1, "total": 1, "accuracy": 1.0 }, "GET_CALORIES": { "correct": 1, "total": 1, "accuracy": 1.0 }, "GET_DISTANCE": { "correct": 1, "total": 1, "accuracy": 1.0 }, "GET_HEART_RATE": { "correct": 1, "total": 1, "accuracy": 1.0 }, "GET_SLEEP": { "correct": 1, "total": 1, "accuracy": 1.0 }, "GET_STEPS": { "correct": 2, "total": 2, "accuracy": 1.0 }, "GET_TIMER_REMAINING": { "correct": 1, "total": 1, "accuracy": 1.0 }, "GET_WORKOUT_STATUS": { "correct": 1, "total": 1, "accuracy": 1.0 }, "GET_WORKOUT_SUMMARY": { "correct": 1, "total": 1, "accuracy": 1.0 }, "LAP_STOPWATCH": { "correct": 1, "total": 1, "accuracy": 1.0 }, "LIST_ALARMS": { "correct": 1, "total": 1, "accuracy": 1.0 }, "LOG_NAP": { "correct": 1, "total": 1, "accuracy": 1.0 }, "MEASURE_HEART_RATE": { "correct": 1, "total": 1, "accuracy": 1.0 }, "MUTE_REMINDERS": { "correct": 1, "total": 1, "accuracy": 1.0 }, "NONE": { "correct": 2, "total": 2, "accuracy": 1.0 }, "PAUSE_TIMER": { "correct": 1, "total": 1, "accuracy": 1.0 }, "PAUSE_WORKOUT": { "correct": 1, "total": 1, "accuracy": 1.0 }, "RESET_STOPWATCH": { "correct": 1, "total": 1, "accuracy": 1.0 }, "RESUME_WORKOUT": { "correct": 1, "total": 1, "accuracy": 1.0 }, "SET_ACTIVE_GOAL": { "correct": 1, "total": 1, "accuracy": 1.0 }, "SET_ALARM": { "correct": 1, "total": 1, "accuracy": 1.0 }, "SET_CALORIE_GOAL": { "correct": 1, "total": 1, "accuracy": 1.0 }, "SET_REMINDER": { "correct": 1, "total": 1, "accuracy": 1.0 }, "SET_STEP_GOAL": { "correct": 1, "total": 1, "accuracy": 1.0 }, "SNOOZE_ALARM": { "correct": 1, "total": 1, "accuracy": 1.0 }, "START_STOPWATCH": { "correct": 1, "total": 1, "accuracy": 1.0 }, "START_TIMER": { "correct": 2, "total": 2, "accuracy": 1.0 }, "START_WORKOUT": { "correct": 1, "total": 1, "accuracy": 1.0 }, "STOP_WORKOUT": { "correct": 1, "total": 1, "accuracy": 1.0 } } } ] }