{ "domain": "support", "seed": 0, "selected_epoch": 3, "test": { "cases": 2166, "accuracy": 0.9976915974145891, "macro_f1": 0.9973333494620968, "per_class": { "cancel_order": { "precision": 0.96875, "recall": 1.0, "f1": 0.9841269841269841, "support": 31 }, "change_order": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 71 }, "change_shipping_address": { "precision": 1.0, "recall": 0.9893617021276596, "f1": 0.9946524064171123, "support": 94 }, "check_cancellation_fee": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 45 }, "check_invoice": { "precision": 0.9895833333333334, "recall": 1.0, "f1": 0.9947643979057592, "support": 95 }, "check_payment_methods": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 58 }, "check_refund_policy": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 122 }, "complaint": { "precision": 0.9915966386554622, "recall": 1.0, "f1": 0.9957805907172996, "support": 118 }, "contact_customer_service": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 89 }, "contact_human_agent": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 70 }, "create_account": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 70 }, "delete_account": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 92 }, "delivery_options": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 65 }, "delivery_period": { "precision": 1.0, "recall": 0.9907407407407407, "f1": 0.9953488372093023, "support": 108 }, "edit_account": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 66 }, "get_invoice": { "precision": 1.0, "recall": 0.9883720930232558, "f1": 0.9941520467836257, "support": 86 }, "get_refund": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 93 }, "newsletter_subscription": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 34 }, "payment_issue": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 110 }, "place_order": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 96 }, "recover_password": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 92 }, "registration_problems": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 73 }, "review": { "precision": 1.0, "recall": 0.9895833333333334, "f1": 0.9947643979057592, "support": 96 }, "set_up_shipping_address": { "precision": 0.9852941176470589, "recall": 1.0, "f1": 0.9925925925925926, "support": 67 }, "switch_account": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 101 }, "track_order": { "precision": 0.9818181818181818, "recall": 0.9818181818181818, "f1": 0.9818181818181818, "support": 55 }, "track_refund": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 69 } }, "confusion": { "cancel_order": { "cancel_order": 31, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "change_order": { "cancel_order": 0, "change_order": 71, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "change_shipping_address": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 93, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 1, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_cancellation_fee": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 45, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_invoice": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 95, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_payment_methods": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 58, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_refund_policy": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 122, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "complaint": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 118, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "contact_customer_service": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 89, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "contact_human_agent": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 70, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "create_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 70, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delete_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 92, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delivery_options": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 65, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delivery_period": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 107, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 1, "track_refund": 0, "INVALID": 0 }, "edit_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 66, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "get_invoice": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 1, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 85, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "get_refund": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 93, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "newsletter_subscription": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 34, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "payment_issue": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 110, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "place_order": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 96, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "recover_password": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 92, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "registration_problems": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 73, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "review": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 1, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 95, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "set_up_shipping_address": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 67, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "switch_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 101, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "track_order": { "cancel_order": 1, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 54, "track_refund": 0, "INVALID": 0 }, "track_refund": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 69, "INVALID": 0 } }, "invalid": 0, "latency": { "requests": 2166, "timed_requests": 2166, "p50_ms": 9.702840994577855, "p95_ms": 9.829692528001033, "p99_ms": 10.153670504223554, "total_request_seconds": 21.02225886710221, "attempted_decisions_per_second": 103.0336470354087 } }, "base_laya": { "cases": 2166, "accuracy": 0.971375807940905, "macro_f1": 0.9728464468683354, "per_class": { "cancel_order": { "precision": 0.9393939393939394, "recall": 1.0, "f1": 0.96875, "support": 31 }, "change_order": { "precision": 0.9857142857142858, "recall": 0.971830985915493, "f1": 0.9787234042553191, "support": 71 }, "change_shipping_address": { "precision": 0.96875, "recall": 0.9893617021276596, "f1": 0.9789473684210527, "support": 94 }, "check_cancellation_fee": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 45 }, "check_invoice": { "precision": 0.93, "recall": 0.9789473684210527, "f1": 0.9538461538461539, "support": 95 }, "check_payment_methods": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 58 }, "check_refund_policy": { "precision": 1.0, "recall": 0.9016393442622951, "f1": 0.9482758620689655, "support": 122 }, "complaint": { "precision": 1.0, "recall": 0.9745762711864406, "f1": 0.9871244635193133, "support": 118 }, "contact_customer_service": { "precision": 0.978021978021978, "recall": 1.0, "f1": 0.9888888888888889, "support": 89 }, "contact_human_agent": { "precision": 1.0, "recall": 0.9428571428571428, "f1": 0.9705882352941176, "support": 70 }, "create_account": { "precision": 0.9452054794520548, "recall": 0.9857142857142858, "f1": 0.965034965034965, "support": 70 }, "delete_account": { "precision": 1.0, "recall": 0.9891304347826086, "f1": 0.994535519125683, "support": 92 }, "delivery_options": { "precision": 0.9558823529411765, "recall": 1.0, "f1": 0.9774436090225563, "support": 65 }, "delivery_period": { "precision": 1.0, "recall": 0.9722222222222222, "f1": 0.9859154929577465, "support": 108 }, "edit_account": { "precision": 0.9552238805970149, "recall": 0.9696969696969697, "f1": 0.9624060150375939, "support": 66 }, "get_invoice": { "precision": 0.9746835443037974, "recall": 0.8953488372093024, "f1": 0.9333333333333333, "support": 86 }, "get_refund": { "precision": 0.7560975609756098, "recall": 1.0, "f1": 0.8611111111111112, "support": 93 }, "newsletter_subscription": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 34 }, "payment_issue": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 110 }, "place_order": { "precision": 0.979381443298969, "recall": 0.9895833333333334, "f1": 0.9844559585492227, "support": 96 }, "recover_password": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 92 }, "registration_problems": { "precision": 1.0, "recall": 0.9726027397260274, "f1": 0.9861111111111112, "support": 73 }, "review": { "precision": 1.0, "recall": 0.9791666666666666, "f1": 0.9894736842105263, "support": 96 }, "set_up_shipping_address": { "precision": 1.0, "recall": 0.9253731343283582, "f1": 0.9612403100775194, "support": 67 }, "switch_account": { "precision": 1.0, "recall": 0.9603960396039604, "f1": 0.9797979797979798, "support": 101 }, "track_order": { "precision": 0.9473684210526315, "recall": 0.9818181818181818, "f1": 0.9642857142857143, "support": 55 }, "track_refund": { "precision": 1.0, "recall": 0.8985507246376812, "f1": 0.9465648854961832, "support": 69 } }, "confusion": { "cancel_order": { "cancel_order": 31, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "change_order": { "cancel_order": 0, "change_order": 69, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 2, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "change_shipping_address": { "cancel_order": 0, "change_order": 1, "change_shipping_address": 93, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_cancellation_fee": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 45, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_invoice": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 93, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 2, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_payment_methods": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 58, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_refund_policy": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 110, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 12, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "complaint": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 115, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 3, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "contact_customer_service": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 89, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "contact_human_agent": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 2, "contact_human_agent": 66, "create_account": 1, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 1, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "create_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 69, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 1, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delete_account": { "cancel_order": 1, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 91, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delivery_options": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 65, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delivery_period": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 105, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 3, "track_refund": 0, "INVALID": 0 }, "edit_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 1, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 64, "get_invoice": 0, "get_refund": 1, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "get_invoice": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 7, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 1, "delivery_period": 0, "edit_account": 0, "get_invoice": 77, "get_refund": 1, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "get_refund": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 93, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "newsletter_subscription": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 34, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "payment_issue": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 110, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "place_order": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 1, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 95, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "recover_password": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 92, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "registration_problems": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 2, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 71, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "review": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 2, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 94, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "set_up_shipping_address": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 3, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 2, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 62, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "switch_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 3, "get_invoice": 0, "get_refund": 1, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 97, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "track_order": { "cancel_order": 1, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 54, "track_refund": 0, "INVALID": 0 }, "track_refund": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 7, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 62, "INVALID": 0 } }, "invalid": 0, "latency": { "requests": 2166, "timed_requests": 2166, "p50_ms": 9.654734516516328, "p95_ms": 9.890471235848963, "p99_ms": 12.53618086921051, "total_request_seconds": 21.109110348508693, "attempted_decisions_per_second": 102.60972462787957 } }, "same_dataset_finetune": { "cases": 2166, "accuracy": 0.9995383194829178, "macro_f1": 0.999608029789736, "per_class": { "cancel_order": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 31 }, "change_order": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 71 }, "change_shipping_address": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 94 }, "check_cancellation_fee": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 45 }, "check_invoice": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 95 }, "check_payment_methods": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 58 }, "check_refund_policy": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 122 }, "complaint": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 118 }, "contact_customer_service": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 89 }, "contact_human_agent": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 70 }, "create_account": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 70 }, "delete_account": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 92 }, "delivery_options": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 65 }, "delivery_period": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 108 }, "edit_account": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 66 }, "get_invoice": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 86 }, "get_refund": { "precision": 0.9893617021276596, "recall": 1.0, "f1": 0.9946524064171123, "support": 93 }, "newsletter_subscription": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 34 }, "payment_issue": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 110 }, "place_order": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 96 }, "recover_password": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 92 }, "registration_problems": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 73 }, "review": { "precision": 1.0, "recall": 0.9895833333333334, "f1": 0.9947643979057592, "support": 96 }, "set_up_shipping_address": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 67 }, "switch_account": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 101 }, "track_order": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 55 }, "track_refund": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 69 } }, "confusion": { "cancel_order": { "cancel_order": 31, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "change_order": { "cancel_order": 0, "change_order": 71, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "change_shipping_address": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 94, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_cancellation_fee": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 45, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_invoice": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 95, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_payment_methods": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 58, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_refund_policy": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 122, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "complaint": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 118, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "contact_customer_service": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 89, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "contact_human_agent": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 70, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "create_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 70, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delete_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 92, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delivery_options": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 65, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delivery_period": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 108, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "edit_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 66, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "get_invoice": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 86, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "get_refund": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 93, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "newsletter_subscription": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 34, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "payment_issue": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 110, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "place_order": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 96, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "recover_password": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 92, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "registration_problems": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 73, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "review": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 1, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 95, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "set_up_shipping_address": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 67, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "switch_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 101, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "track_order": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 55, "track_refund": 0, "INVALID": 0 }, "track_refund": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 69, "INVALID": 0 } }, "out_of_task_or_abstained_predictions": 0, "model": { "repo_id": "learn-abc/magicSupport-intent-classifier", "dataset": "bitext/Bitext-customer-support-llm-chatbot-training-dataset", "max_length": 512, "label_policy": "Preserve all 28 output classes. FALLBACK counts as an incorrect/abstained decision on the 27-intent in-domain test; never remove its logit.", "split_status": "Published training row IDs unavailable; overlap with our custom split is unknown and likely. Same-input source-corpus comparison only until independent evaluation.", "mapping": { "0": "FALLBACK", "1": "cancel_order", "2": "change_order", "3": "change_shipping_address", "4": "check_cancellation_fee", "5": "check_invoice", "6": "check_payment_methods", "7": "check_refund_policy", "8": "complaint", "9": "contact_customer_service", "10": "contact_human_agent", "11": "create_account", "12": "delete_account", "13": "delivery_options", "14": "delivery_period", "15": "edit_account", "16": "get_invoice", "17": "get_refund", "18": "newsletter_subscription", "19": "payment_issue", "20": "place_order", "21": "recover_password", "22": "registration_problems", "23": "review", "24": "set_up_shipping_address", "25": "switch_account", "26": "track_order", "27": "track_refund" }, "reported_accuracy": 0.9983, "revision": "724ee4932560d8b8e29cb86321093e8f41188a35", "directory": "data/specializations/comparators/learn-abc--magicSupport-intent-classifier", "id2label": { "0": "FALLBACK", "1": "cancel_order", "2": "change_order", "3": "change_shipping_address", "4": "check_cancellation_fee", "5": "check_invoice", "6": "check_payment_methods", "7": "check_refund_policy", "8": "complaint", "9": "contact_customer_service", "10": "contact_human_agent", "11": "create_account", "12": "delete_account", "13": "delivery_options", "14": "delivery_period", "15": "edit_account", "16": "get_invoice", "17": "get_refund", "18": "newsletter_subscription", "19": "payment_issue", "20": "place_order", "21": "recover_password", "22": "registration_problems", "23": "review", "24": "set_up_shipping_address", "25": "switch_account", "26": "track_order", "27": "track_refund" }, "model_card_url": "https://huggingface.co/learn-abc/magicSupport-intent-classifier/blob/724ee4932560d8b8e29cb86321093e8f41188a35/README.md", "status": "selected; local evaluation pending", "metadata_sha256": { "README.md": "f36ac038067cb7db90eec4fb9b21a8431b8088cd8d4510cb63c374c8ba62cda2", "config.json": "0b7ecc83246c238d69b221694dd4623d7f6f9ba6d507d10a942387948309a3e5" } }, "manifest": { "format_version": 1, "sha256": "52130a5122b98c6e61834902c472c4f8c541193af060317f01e3afc58a6458a5", "cases": 2166, "decisions": 2166, "profile": "specialization-v1", "source": { "id": "bitext/Bitext-customer-support-llm-chatbot-training-dataset", "revision": "430d1a89bd93bd1fa23c16f29dd53e73f0087443", "license_metadata": "cdla-sharing-1.0", "url": "https://huggingface.co/datasets/bitext/Bitext-customer-support-llm-chatbot-training-dataset", "metadata_only": true }, "split_seed": 42, "candidate_policy": "all 27 labels, fixed order, no gold-dependent shortlist", "grouping": { "method": "connected components; char_wb TF-IDF 3–5 grams; cosine >= threshold", "threshold": 0.9, "edges": 24581, "groups": 12727, "largest_group": 146, "labels_used_for_similarity": false, "limitation": "source template IDs are unavailable; semantic paraphrase independence is unverified" }, "source_file_sha256": { "Bitext_Sample_Customer_Support_Training_Dataset_27K_responses-v11.csv": "6f81102b0100b97b8468eb04368033a23206bf1fde9d53500d5806ec1001a434" }, "role": "test", "label_counts": { "cancel_order": 31, "change_order": 71, "change_shipping_address": 94, "check_cancellation_fee": 45, "check_invoice": 95, "check_payment_methods": 58, "check_refund_policy": 122, "complaint": 118, "contact_customer_service": 89, "contact_human_agent": 70, "create_account": 70, "delete_account": 92, "delivery_options": 65, "delivery_period": 108, "edit_account": 66, "get_invoice": 86, "get_refund": 93, "newsletter_subscription": 34, "payment_issue": 110, "place_order": 96, "recover_password": 92, "registration_problems": 73, "review": 96, "set_up_shipping_address": 67, "switch_account": 101, "track_order": 55, "track_refund": 69 } }, "device": "cuda:1", "dtype": "float32", "batch_size": 32, "truncated_inputs": 0, "successful_predictions": 2166, "forward_seconds": 0.5104441600269638, "forward_timing_includes_warmup": true, "same_input_comparison": true, "common_unseen_test_verified": false, "parameters": 109503772, "weights_sha256": "3e10e59330cf2800350abf09ce08a9ff9942251d55499deb00b08bb34b758c03" }, "simple_comparator": { "cases": 2166, "accuracy": 0.9921514312096029, "macro_f1": 0.9916990546268141, "per_class": { "cancel_order": { "precision": 1.0, "recall": 0.967741935483871, "f1": 0.9836065573770492, "support": 31 }, "change_order": { "precision": 0.9705882352941176, "recall": 0.9295774647887324, "f1": 0.9496402877697842, "support": 71 }, "change_shipping_address": { "precision": 0.9894736842105263, "recall": 1.0, "f1": 0.9947089947089947, "support": 94 }, "check_cancellation_fee": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 45 }, "check_invoice": { "precision": 0.9489795918367347, "recall": 0.9789473684210527, "f1": 0.9637305699481865, "support": 95 }, "check_payment_methods": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 58 }, "check_refund_policy": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 122 }, "complaint": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 118 }, "contact_customer_service": { "precision": 1.0, "recall": 0.9887640449438202, "f1": 0.9943502824858758, "support": 89 }, "contact_human_agent": { "precision": 0.9859154929577465, "recall": 1.0, "f1": 0.9929078014184397, "support": 70 }, "create_account": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 70 }, "delete_account": { "precision": 0.9787234042553191, "recall": 1.0, "f1": 0.989247311827957, "support": 92 }, "delivery_options": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 65 }, "delivery_period": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 108 }, "edit_account": { "precision": 0.9565217391304348, "recall": 1.0, "f1": 0.9777777777777777, "support": 66 }, "get_invoice": { "precision": 0.9759036144578314, "recall": 0.9418604651162791, "f1": 0.9585798816568047, "support": 86 }, "get_refund": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 93 }, "newsletter_subscription": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 34 }, "payment_issue": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 110 }, "place_order": { "precision": 0.9896907216494846, "recall": 1.0, "f1": 0.9948186528497409, "support": 96 }, "recover_password": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 92 }, "registration_problems": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 73 }, "review": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 96 }, "set_up_shipping_address": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 67 }, "switch_account": { "precision": 1.0, "recall": 0.9900990099009901, "f1": 0.9950248756218906, "support": 101 }, "track_order": { "precision": 1.0, "recall": 0.9636363636363636, "f1": 0.9814814814814815, "support": 55 }, "track_refund": { "precision": 1.0, "recall": 1.0, "f1": 1.0, "support": 69 } }, "confusion": { "cancel_order": { "cancel_order": 30, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 1, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "change_order": { "cancel_order": 0, "change_order": 66, "change_shipping_address": 1, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 3, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 1, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "change_shipping_address": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 94, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_cancellation_fee": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 45, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_invoice": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 93, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 2, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_payment_methods": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 58, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "check_refund_policy": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 122, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "complaint": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 118, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "contact_customer_service": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 88, "contact_human_agent": 1, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "contact_human_agent": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 70, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "create_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 70, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delete_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 92, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delivery_options": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 65, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "delivery_period": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 108, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "edit_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 66, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "get_invoice": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 5, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 81, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "get_refund": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 93, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "newsletter_subscription": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 34, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "payment_issue": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 110, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "place_order": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 96, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "recover_password": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 92, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "registration_problems": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 73, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "review": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 96, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "set_up_shipping_address": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 67, "switch_account": 0, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "switch_account": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 1, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 100, "track_order": 0, "track_refund": 0, "INVALID": 0 }, "track_order": { "cancel_order": 0, "change_order": 2, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 53, "track_refund": 0, "INVALID": 0 }, "track_refund": { "cancel_order": 0, "change_order": 0, "change_shipping_address": 0, "check_cancellation_fee": 0, "check_invoice": 0, "check_payment_methods": 0, "check_refund_policy": 0, "complaint": 0, "contact_customer_service": 0, "contact_human_agent": 0, "create_account": 0, "delete_account": 0, "delivery_options": 0, "delivery_period": 0, "edit_account": 0, "get_invoice": 0, "get_refund": 0, "newsletter_subscription": 0, "payment_issue": 0, "place_order": 0, "recover_password": 0, "registration_problems": 0, "review": 0, "set_up_shipping_address": 0, "switch_account": 0, "track_order": 0, "track_refund": 69, "INVALID": 0 } }, "invalid": 0 }, "retention": { "typed-decisions": 0.356, "ag-news": 0.9466666666666667, "emotion": 0.5716666666666667, "boolq": 0.8233333333333334, "sst5": 0.38166666666666665, "prompt-injections": 0.7155172413793104, "massive-intent.en": 0.7733333333333333, "xnli.en": 0.8666666666666667 }, "independent_external_test": false }