File size: 6,435 Bytes
cb96101 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 | """Test requests, sized to land in each bucket. Shared by scripts/make_golden.py and the tests."""
SUPPORT_TASKS = {
"intent": ["order_status", "refund_request", "cancel_subscription", "update_payment", "other"],
"urgency": ["low", "normal", "high"],
}
CASES = [
{
"id": "short-refund",
"bucket": 64,
"text": "My subscription renewed on April 15 for 5,400 yen after the service was already down. "
"Can I get that charge refunded?",
"tasks": SUPPORT_TASKS,
},
{
"id": "laptop-review-aspects",
"bucket": 64,
"text": "Battery dies before lunch, but the keyboard and the screen are the best I have used on a laptop.",
"tasks": {
"sentiment": ["positive", "negative", "mixed", "neutral"],
"aspects": {"labels": ["battery", "keyboard", "screen", "camera", "price", "support"],
"multi_label": True, "cls_threshold": 0.4},
},
},
{
"id": "compliance-email-3-heads",
"bucket": 128,
"text": "From: compliance@group.example\nSubject: Protocol update, action required today\n\n"
"Please confirm the new retention rule is applied before Friday's audit.",
"tasks": {
"intent": ["fyi", "request", "approval", "complaint", "newsletter", "security_alert"],
"urgency": ["low", "normal", "high", "critical"],
"route": ["support", "billing", "legal", "security", "finance", "archive"],
},
},
{
"id": "descriptions-and-prompt",
"bucket": 128,
"text": "My subscription renewed on April 15 for 5,400 yen after the service was already down. "
"Can I get that charge refunded?",
"tasks": {
"intent": {"labels": {
"order_status": "The customer asks where an order is",
"refund_request": "The customer wants money back",
"cancel_subscription": "The customer wants to stop a recurring plan",
"update_payment": "The customer wants to change a card or billing method",
"other": "None of the above",
}},
"urgency": {"labels": ["low", "normal", "high"], "prompt": "How urgently should support respond?"},
},
},
{
"id": "hotel-complaint-4-heads",
"bucket": 256,
"text": "Hello, I'm writing from room 1408. The air conditioning stopped working yesterday afternoon "
"and the room is now above 30 degrees, so neither of us could sleep last night. We called the "
"front desk twice and were told a technician would come, but nobody showed up. We have a "
"conference presentation tomorrow morning and cannot spend another night like this. Please "
"either fix it within the next two hours or move us to another room tonight. If neither is "
"possible we will check out early and expect the remaining nights to be refunded. We would "
"also like the incidentals hold on our card to be released, since we were told at check-in "
"that it would only last one day. Thank you.",
"tasks": {
"intent": ["maintenance", "room_change", "checkout", "billing", "complaint", "amenity_request"],
"priority": ["low", "normal", "high", "urgent"],
"needs_human": ["yes", "no"],
"topics": {"labels": ["hvac", "billing", "housekeeping", "noise", "safety"],
"multi_label": True, "cls_threshold": 0.4},
},
},
{
"id": "incident-report-long",
"bucket": 512,
"text": "Incident summary for the payments platform, prepared for the weekly operations review. "
"On Tuesday at 09:12 UTC the card authorization service began returning timeouts for roughly "
"one in five requests. The on-call engineer was paged four minutes later by the latency alert "
"and started investigating the database tier, which showed normal load. At 09:31 the team noticed "
"that a configuration change deployed the previous evening had lowered the connection pool size "
"for the authorization service from two hundred to twenty connections. Under the morning traffic "
"peak the pool was exhausted, requests queued, and the upstream gateway gave up after its "
"three second deadline. The change had been reviewed, but the reviewer assumed the value was "
"per worker rather than per service, and the staging environment does not receive enough "
"traffic to reveal the problem. The configuration was rolled back at 09:44 and error rates "
"returned to normal within two minutes. In total about eleven thousand authorizations failed; "
"most customers retried successfully, but around four hundred orders were abandoned. No card "
"data was exposed and there is no indication of any security issue. Follow-up actions: add a "
"load test that replays production traffic volumes against staging before configuration "
"changes to the authorization service are approved; make pool sizes explicit about their "
"scope in the configuration schema; add an alert on connection pool saturation so the "
"on-call engineer is pointed at the right component immediately; and review whether the "
"gateway deadline should be longer for authorization calls. Finance has asked for an estimate "
"of lost revenue from abandoned orders by the end of the week, and customer support has "
"prepared a short statement in case merchants ask about the failed transactions. The service "
"owner will present these actions at the architecture forum next Thursday and track them to "
"completion in the reliability backlog.",
"tasks": {
"doc_type": ["incident_report", "feature_request", "meeting_notes", "security_advisory", "marketing"],
"severity": ["sev1", "sev2", "sev3", "sev4"],
"teams": {"labels": ["payments", "security", "finance", "support", "marketing", "legal"],
"multi_label": True, "cls_threshold": 0.4},
},
},
]
|