diff --git "a/metrics.json" "b/metrics.json" new file mode 100644--- /dev/null +++ "b/metrics.json" @@ -0,0 +1,4107 @@ +{ + "domain": "support", + "seed": 0, + "selected_epoch": 3, + "test": { + "cases": 2166, + "accuracy": 0.9976915974145891, + "macro_f1": 0.9973333494620968, + "per_class": { + "cancel_order": { + "precision": 0.96875, + "recall": 1.0, + "f1": 0.9841269841269841, + "support": 31 + }, + "change_order": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 71 + }, + "change_shipping_address": { + "precision": 1.0, + "recall": 0.9893617021276596, + "f1": 0.9946524064171123, + "support": 94 + }, + "check_cancellation_fee": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 45 + }, + "check_invoice": { + "precision": 0.9895833333333334, + "recall": 1.0, + "f1": 0.9947643979057592, + "support": 95 + }, + "check_payment_methods": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 58 + }, + "check_refund_policy": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 122 + }, + "complaint": { + "precision": 0.9915966386554622, + "recall": 1.0, + "f1": 0.9957805907172996, + "support": 118 + }, + "contact_customer_service": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 89 + }, + "contact_human_agent": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 70 + }, + "create_account": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 70 + }, + "delete_account": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 92 + }, + "delivery_options": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 65 + }, + "delivery_period": { + "precision": 1.0, + "recall": 0.9907407407407407, + "f1": 0.9953488372093023, + "support": 108 + }, + "edit_account": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 66 + }, + "get_invoice": { + "precision": 1.0, + "recall": 0.9883720930232558, + "f1": 0.9941520467836257, + "support": 86 + }, + "get_refund": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 93 + }, + "newsletter_subscription": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 34 + }, + "payment_issue": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 110 + }, + "place_order": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 96 + }, + "recover_password": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 92 + }, + "registration_problems": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 73 + }, + "review": { + "precision": 1.0, + "recall": 0.9895833333333334, + "f1": 0.9947643979057592, + "support": 96 + }, + "set_up_shipping_address": { + "precision": 0.9852941176470589, + "recall": 1.0, + "f1": 0.9925925925925926, + "support": 67 + }, + "switch_account": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 101 + }, + "track_order": { + "precision": 0.9818181818181818, + "recall": 0.9818181818181818, + "f1": 0.9818181818181818, + "support": 55 + }, + "track_refund": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 69 + } + }, + "confusion": { + "cancel_order": { + "cancel_order": 31, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "change_order": { + "cancel_order": 0, + "change_order": 71, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "change_shipping_address": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 93, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 1, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_cancellation_fee": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 45, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_invoice": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 95, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_payment_methods": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 58, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_refund_policy": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 122, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "complaint": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 118, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "contact_customer_service": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 89, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "contact_human_agent": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 70, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "create_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 70, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delete_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 92, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delivery_options": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 65, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delivery_period": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 107, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 1, + "track_refund": 0, + "INVALID": 0 + }, + "edit_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 66, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "get_invoice": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 1, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 85, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "get_refund": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 93, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "newsletter_subscription": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 34, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "payment_issue": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 110, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "place_order": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 96, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "recover_password": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 92, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "registration_problems": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 73, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "review": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 1, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 95, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "set_up_shipping_address": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 67, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "switch_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 101, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "track_order": { + "cancel_order": 1, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 54, + "track_refund": 0, + "INVALID": 0 + }, + "track_refund": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 69, + "INVALID": 0 + } + }, + "invalid": 0, + "latency": { + "requests": 2166, + "timed_requests": 2166, + "p50_ms": 9.702840994577855, + "p95_ms": 9.829692528001033, + "p99_ms": 10.153670504223554, + "total_request_seconds": 21.02225886710221, + "attempted_decisions_per_second": 103.0336470354087 + } + }, + "base_laya": { + "cases": 2166, + "accuracy": 0.971375807940905, + "macro_f1": 0.9728464468683354, + "per_class": { + "cancel_order": { + "precision": 0.9393939393939394, + "recall": 1.0, + "f1": 0.96875, + "support": 31 + }, + "change_order": { + "precision": 0.9857142857142858, + "recall": 0.971830985915493, + "f1": 0.9787234042553191, + "support": 71 + }, + "change_shipping_address": { + "precision": 0.96875, + "recall": 0.9893617021276596, + "f1": 0.9789473684210527, + "support": 94 + }, + "check_cancellation_fee": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 45 + }, + "check_invoice": { + "precision": 0.93, + "recall": 0.9789473684210527, + "f1": 0.9538461538461539, + "support": 95 + }, + "check_payment_methods": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 58 + }, + "check_refund_policy": { + "precision": 1.0, + "recall": 0.9016393442622951, + "f1": 0.9482758620689655, + "support": 122 + }, + "complaint": { + "precision": 1.0, + "recall": 0.9745762711864406, + "f1": 0.9871244635193133, + "support": 118 + }, + "contact_customer_service": { + "precision": 0.978021978021978, + "recall": 1.0, + "f1": 0.9888888888888889, + "support": 89 + }, + "contact_human_agent": { + "precision": 1.0, + "recall": 0.9428571428571428, + "f1": 0.9705882352941176, + "support": 70 + }, + "create_account": { + "precision": 0.9452054794520548, + "recall": 0.9857142857142858, + "f1": 0.965034965034965, + "support": 70 + }, + "delete_account": { + "precision": 1.0, + "recall": 0.9891304347826086, + "f1": 0.994535519125683, + "support": 92 + }, + "delivery_options": { + "precision": 0.9558823529411765, + "recall": 1.0, + "f1": 0.9774436090225563, + "support": 65 + }, + "delivery_period": { + "precision": 1.0, + "recall": 0.9722222222222222, + "f1": 0.9859154929577465, + "support": 108 + }, + "edit_account": { + "precision": 0.9552238805970149, + "recall": 0.9696969696969697, + "f1": 0.9624060150375939, + "support": 66 + }, + "get_invoice": { + "precision": 0.9746835443037974, + "recall": 0.8953488372093024, + "f1": 0.9333333333333333, + "support": 86 + }, + "get_refund": { + "precision": 0.7560975609756098, + "recall": 1.0, + "f1": 0.8611111111111112, + "support": 93 + }, + "newsletter_subscription": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 34 + }, + "payment_issue": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 110 + }, + "place_order": { + "precision": 0.979381443298969, + "recall": 0.9895833333333334, + "f1": 0.9844559585492227, + "support": 96 + }, + "recover_password": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 92 + }, + "registration_problems": { + "precision": 1.0, + "recall": 0.9726027397260274, + "f1": 0.9861111111111112, + "support": 73 + }, + "review": { + "precision": 1.0, + "recall": 0.9791666666666666, + "f1": 0.9894736842105263, + "support": 96 + }, + "set_up_shipping_address": { + "precision": 1.0, + "recall": 0.9253731343283582, + "f1": 0.9612403100775194, + "support": 67 + }, + "switch_account": { + "precision": 1.0, + "recall": 0.9603960396039604, + "f1": 0.9797979797979798, + "support": 101 + }, + "track_order": { + "precision": 0.9473684210526315, + "recall": 0.9818181818181818, + "f1": 0.9642857142857143, + "support": 55 + }, + "track_refund": { + "precision": 1.0, + "recall": 0.8985507246376812, + "f1": 0.9465648854961832, + "support": 69 + } + }, + "confusion": { + "cancel_order": { + "cancel_order": 31, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "change_order": { + "cancel_order": 0, + "change_order": 69, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 2, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "change_shipping_address": { + "cancel_order": 0, + "change_order": 1, + "change_shipping_address": 93, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_cancellation_fee": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 45, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_invoice": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 93, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 2, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_payment_methods": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 58, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_refund_policy": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 110, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 12, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "complaint": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 115, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 3, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "contact_customer_service": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 89, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "contact_human_agent": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 2, + "contact_human_agent": 66, + "create_account": 1, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 1, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "create_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 69, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 1, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delete_account": { + "cancel_order": 1, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 91, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delivery_options": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 65, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delivery_period": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 105, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 3, + "track_refund": 0, + "INVALID": 0 + }, + "edit_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 1, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 64, + "get_invoice": 0, + "get_refund": 1, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "get_invoice": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 7, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 1, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 77, + "get_refund": 1, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "get_refund": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 93, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "newsletter_subscription": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 34, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "payment_issue": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 110, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "place_order": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 1, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 95, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "recover_password": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 92, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "registration_problems": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 2, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 71, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "review": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 2, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 94, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "set_up_shipping_address": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 3, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 2, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 62, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "switch_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 3, + "get_invoice": 0, + "get_refund": 1, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 97, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "track_order": { + "cancel_order": 1, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 54, + "track_refund": 0, + "INVALID": 0 + }, + "track_refund": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 7, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 62, + "INVALID": 0 + } + }, + "invalid": 0, + "latency": { + "requests": 2166, + "timed_requests": 2166, + "p50_ms": 9.654734516516328, + "p95_ms": 9.890471235848963, + "p99_ms": 12.53618086921051, + "total_request_seconds": 21.109110348508693, + "attempted_decisions_per_second": 102.60972462787957 + } + }, + "same_dataset_finetune": { + "cases": 2166, + "accuracy": 0.9995383194829178, + "macro_f1": 0.999608029789736, + "per_class": { + "cancel_order": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 31 + }, + "change_order": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 71 + }, + "change_shipping_address": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 94 + }, + "check_cancellation_fee": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 45 + }, + "check_invoice": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 95 + }, + "check_payment_methods": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 58 + }, + "check_refund_policy": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 122 + }, + "complaint": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 118 + }, + "contact_customer_service": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 89 + }, + "contact_human_agent": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 70 + }, + "create_account": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 70 + }, + "delete_account": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 92 + }, + "delivery_options": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 65 + }, + "delivery_period": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 108 + }, + "edit_account": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 66 + }, + "get_invoice": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 86 + }, + "get_refund": { + "precision": 0.9893617021276596, + "recall": 1.0, + "f1": 0.9946524064171123, + "support": 93 + }, + "newsletter_subscription": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 34 + }, + "payment_issue": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 110 + }, + "place_order": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 96 + }, + "recover_password": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 92 + }, + "registration_problems": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 73 + }, + "review": { + "precision": 1.0, + "recall": 0.9895833333333334, + "f1": 0.9947643979057592, + "support": 96 + }, + "set_up_shipping_address": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 67 + }, + "switch_account": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 101 + }, + "track_order": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 55 + }, + "track_refund": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 69 + } + }, + "confusion": { + "cancel_order": { + "cancel_order": 31, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "change_order": { + "cancel_order": 0, + "change_order": 71, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "change_shipping_address": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 94, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_cancellation_fee": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 45, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_invoice": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 95, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_payment_methods": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 58, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_refund_policy": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 122, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "complaint": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 118, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "contact_customer_service": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 89, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "contact_human_agent": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 70, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "create_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 70, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delete_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 92, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delivery_options": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 65, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delivery_period": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 108, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "edit_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 66, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "get_invoice": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 86, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "get_refund": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 93, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "newsletter_subscription": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 34, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "payment_issue": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 110, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "place_order": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 96, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "recover_password": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 92, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "registration_problems": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 73, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "review": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 1, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 95, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "set_up_shipping_address": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 67, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "switch_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 101, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "track_order": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 55, + "track_refund": 0, + "INVALID": 0 + }, + "track_refund": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 69, + "INVALID": 0 + } + }, + "out_of_task_or_abstained_predictions": 0, + "model": { + "repo_id": "learn-abc/magicSupport-intent-classifier", + "dataset": "bitext/Bitext-customer-support-llm-chatbot-training-dataset", + "max_length": 512, + "label_policy": "Preserve all 28 output classes. FALLBACK counts as an incorrect/abstained decision on the 27-intent in-domain test; never remove its logit.", + "split_status": "Published training row IDs unavailable; overlap with our custom split is unknown and likely. Same-input source-corpus comparison only until independent evaluation.", + "mapping": { + "0": "FALLBACK", + "1": "cancel_order", + "2": "change_order", + "3": "change_shipping_address", + "4": "check_cancellation_fee", + "5": "check_invoice", + "6": "check_payment_methods", + "7": "check_refund_policy", + "8": "complaint", + "9": "contact_customer_service", + "10": "contact_human_agent", + "11": "create_account", + "12": "delete_account", + "13": "delivery_options", + "14": "delivery_period", + "15": "edit_account", + "16": "get_invoice", + "17": "get_refund", + "18": "newsletter_subscription", + "19": "payment_issue", + "20": "place_order", + "21": "recover_password", + "22": "registration_problems", + "23": "review", + "24": "set_up_shipping_address", + "25": "switch_account", + "26": "track_order", + "27": "track_refund" + }, + "reported_accuracy": 0.9983, + "revision": "724ee4932560d8b8e29cb86321093e8f41188a35", + "directory": "data/specializations/comparators/learn-abc--magicSupport-intent-classifier", + "id2label": { + "0": "FALLBACK", + "1": "cancel_order", + "2": "change_order", + "3": "change_shipping_address", + "4": "check_cancellation_fee", + "5": "check_invoice", + "6": "check_payment_methods", + "7": "check_refund_policy", + "8": "complaint", + "9": "contact_customer_service", + "10": "contact_human_agent", + "11": "create_account", + "12": "delete_account", + "13": "delivery_options", + "14": "delivery_period", + "15": "edit_account", + "16": "get_invoice", + "17": "get_refund", + "18": "newsletter_subscription", + "19": "payment_issue", + "20": "place_order", + "21": "recover_password", + "22": "registration_problems", + "23": "review", + "24": "set_up_shipping_address", + "25": "switch_account", + "26": "track_order", + "27": "track_refund" + }, + "model_card_url": "https://huggingface.co/learn-abc/magicSupport-intent-classifier/blob/724ee4932560d8b8e29cb86321093e8f41188a35/README.md", + "status": "selected; local evaluation pending", + "metadata_sha256": { + "README.md": "f36ac038067cb7db90eec4fb9b21a8431b8088cd8d4510cb63c374c8ba62cda2", + "config.json": "0b7ecc83246c238d69b221694dd4623d7f6f9ba6d507d10a942387948309a3e5" + } + }, + "manifest": { + "format_version": 1, + "sha256": "52130a5122b98c6e61834902c472c4f8c541193af060317f01e3afc58a6458a5", + "cases": 2166, + "decisions": 2166, + "profile": "specialization-v1", + "source": { + "id": "bitext/Bitext-customer-support-llm-chatbot-training-dataset", + "revision": "430d1a89bd93bd1fa23c16f29dd53e73f0087443", + "license_metadata": "cdla-sharing-1.0", + "url": "https://huggingface.co/datasets/bitext/Bitext-customer-support-llm-chatbot-training-dataset", + "metadata_only": true + }, + "split_seed": 42, + "candidate_policy": "all 27 labels, fixed order, no gold-dependent shortlist", + "grouping": { + "method": "connected components; char_wb TF-IDF 3–5 grams; cosine >= threshold", + "threshold": 0.9, + "edges": 24581, + "groups": 12727, + "largest_group": 146, + "labels_used_for_similarity": false, + "limitation": "source template IDs are unavailable; semantic paraphrase independence is unverified" + }, + "source_file_sha256": { + "Bitext_Sample_Customer_Support_Training_Dataset_27K_responses-v11.csv": "6f81102b0100b97b8468eb04368033a23206bf1fde9d53500d5806ec1001a434" + }, + "role": "test", + "label_counts": { + "cancel_order": 31, + "change_order": 71, + "change_shipping_address": 94, + "check_cancellation_fee": 45, + "check_invoice": 95, + "check_payment_methods": 58, + "check_refund_policy": 122, + "complaint": 118, + "contact_customer_service": 89, + "contact_human_agent": 70, + "create_account": 70, + "delete_account": 92, + "delivery_options": 65, + "delivery_period": 108, + "edit_account": 66, + "get_invoice": 86, + "get_refund": 93, + "newsletter_subscription": 34, + "payment_issue": 110, + "place_order": 96, + "recover_password": 92, + "registration_problems": 73, + "review": 96, + "set_up_shipping_address": 67, + "switch_account": 101, + "track_order": 55, + "track_refund": 69 + } + }, + "device": "cuda:1", + "dtype": "float32", + "batch_size": 32, + "truncated_inputs": 0, + "successful_predictions": 2166, + "forward_seconds": 0.5104441600269638, + "forward_timing_includes_warmup": true, + "same_input_comparison": true, + "common_unseen_test_verified": false, + "parameters": 109503772, + "weights_sha256": "3e10e59330cf2800350abf09ce08a9ff9942251d55499deb00b08bb34b758c03" + }, + "simple_comparator": { + "cases": 2166, + "accuracy": 0.9921514312096029, + "macro_f1": 0.9916990546268141, + "per_class": { + "cancel_order": { + "precision": 1.0, + "recall": 0.967741935483871, + "f1": 0.9836065573770492, + "support": 31 + }, + "change_order": { + "precision": 0.9705882352941176, + "recall": 0.9295774647887324, + "f1": 0.9496402877697842, + "support": 71 + }, + "change_shipping_address": { + "precision": 0.9894736842105263, + "recall": 1.0, + "f1": 0.9947089947089947, + "support": 94 + }, + "check_cancellation_fee": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 45 + }, + "check_invoice": { + "precision": 0.9489795918367347, + "recall": 0.9789473684210527, + "f1": 0.9637305699481865, + "support": 95 + }, + "check_payment_methods": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 58 + }, + "check_refund_policy": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 122 + }, + "complaint": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 118 + }, + "contact_customer_service": { + "precision": 1.0, + "recall": 0.9887640449438202, + "f1": 0.9943502824858758, + "support": 89 + }, + "contact_human_agent": { + "precision": 0.9859154929577465, + "recall": 1.0, + "f1": 0.9929078014184397, + "support": 70 + }, + "create_account": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 70 + }, + "delete_account": { + "precision": 0.9787234042553191, + "recall": 1.0, + "f1": 0.989247311827957, + "support": 92 + }, + "delivery_options": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 65 + }, + "delivery_period": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 108 + }, + "edit_account": { + "precision": 0.9565217391304348, + "recall": 1.0, + "f1": 0.9777777777777777, + "support": 66 + }, + "get_invoice": { + "precision": 0.9759036144578314, + "recall": 0.9418604651162791, + "f1": 0.9585798816568047, + "support": 86 + }, + "get_refund": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 93 + }, + "newsletter_subscription": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 34 + }, + "payment_issue": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 110 + }, + "place_order": { + "precision": 0.9896907216494846, + "recall": 1.0, + "f1": 0.9948186528497409, + "support": 96 + }, + "recover_password": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 92 + }, + "registration_problems": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 73 + }, + "review": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 96 + }, + "set_up_shipping_address": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 67 + }, + "switch_account": { + "precision": 1.0, + "recall": 0.9900990099009901, + "f1": 0.9950248756218906, + "support": 101 + }, + "track_order": { + "precision": 1.0, + "recall": 0.9636363636363636, + "f1": 0.9814814814814815, + "support": 55 + }, + "track_refund": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "support": 69 + } + }, + "confusion": { + "cancel_order": { + "cancel_order": 30, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 1, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "change_order": { + "cancel_order": 0, + "change_order": 66, + "change_shipping_address": 1, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 3, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 1, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "change_shipping_address": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 94, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_cancellation_fee": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 45, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_invoice": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 93, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 2, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_payment_methods": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 58, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "check_refund_policy": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 122, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "complaint": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 118, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "contact_customer_service": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 88, + "contact_human_agent": 1, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "contact_human_agent": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 70, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "create_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 70, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delete_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 92, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delivery_options": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 65, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "delivery_period": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 108, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "edit_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 66, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "get_invoice": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 5, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 81, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "get_refund": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 93, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "newsletter_subscription": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 34, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "payment_issue": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 110, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "place_order": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 96, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "recover_password": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 92, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "registration_problems": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 73, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "review": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 96, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "set_up_shipping_address": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 67, + "switch_account": 0, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "switch_account": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 1, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 100, + "track_order": 0, + "track_refund": 0, + "INVALID": 0 + }, + "track_order": { + "cancel_order": 0, + "change_order": 2, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 53, + "track_refund": 0, + "INVALID": 0 + }, + "track_refund": { + "cancel_order": 0, + "change_order": 0, + "change_shipping_address": 0, + "check_cancellation_fee": 0, + "check_invoice": 0, + "check_payment_methods": 0, + "check_refund_policy": 0, + "complaint": 0, + "contact_customer_service": 0, + "contact_human_agent": 0, + "create_account": 0, + "delete_account": 0, + "delivery_options": 0, + "delivery_period": 0, + "edit_account": 0, + "get_invoice": 0, + "get_refund": 0, + "newsletter_subscription": 0, + "payment_issue": 0, + "place_order": 0, + "recover_password": 0, + "registration_problems": 0, + "review": 0, + "set_up_shipping_address": 0, + "switch_account": 0, + "track_order": 0, + "track_refund": 69, + "INVALID": 0 + } + }, + "invalid": 0 + }, + "retention": { + "typed-decisions": 0.356, + "ag-news": 0.9466666666666667, + "emotion": 0.5716666666666667, + "boolq": 0.8233333333333334, + "sst5": 0.38166666666666665, + "prompt-injections": 0.7155172413793104, + "massive-intent.en": 0.7733333333333333, + "xnli.en": 0.8666666666666667 + }, + "independent_external_test": false +}