| from __future__ import annotations |
|
|
| from collections import Counter |
| from typing import Any |
|
|
| SEVERITY_MAP = {"low": 1, "medium": 3, "high": 5} |
|
|
| CANONICAL_ATTACK_CATEGORIES = ( |
| "instruction_override", |
| "prompt_leakage", |
| "retrieved_indirect", |
| "evasion_obfuscation", |
| "adaptive_multiturn", |
| "benign_control", |
| ) |
|
|
| SUCCESS_DEFINITION_IDS = ( |
| "s1_instruction_override", |
| "s2_prompt_leakage", |
| "s3_retrieved_indirect", |
| "s4_evasion_obfuscation", |
| "s5_adaptive_multiturn", |
| "s6_benign_control", |
| ) |
|
|
| MAPPING_TABLE: dict[str, tuple[str, str, str]] = { |
| "retrieved_content_injection": ("retrieved_indirect", "s3_retrieved_indirect", "high"), |
| "benign_control": ("benign_control", "s6_benign_control", "high"), |
| "system_prompt_extraction": ("prompt_leakage", "s2_prompt_leakage", "high"), |
| "prompt_leakage": ("prompt_leakage", "s2_prompt_leakage", "high"), |
| "direct_instruction_override": ("instruction_override", "s1_instruction_override", "high"), |
| "instruction_override": ("instruction_override", "s1_instruction_override", "high"), |
| "compound_instruction_attack": ("instruction_override", "s1_instruction_override", "medium"), |
| "sandwich_defense_bypass": ("evasion_obfuscation", "s4_evasion_obfuscation", "high"), |
| "blacklist_evasion": ("evasion_obfuscation", "s4_evasion_obfuscation", "high"), |
| "xml_escape_evasion": ("evasion_obfuscation", "s4_evasion_obfuscation", "high"), |
| "restricted_character_bypass": ("evasion_obfuscation", "s4_evasion_obfuscation", "high"), |
| "emoji_only_bypass": ("evasion_obfuscation", "s4_evasion_obfuscation", "high"), |
| "adaptive_attack": ("adaptive_multiturn", "s5_adaptive_multiturn", "high"), |
| } |
|
|
| EVALUATION_MODE_BY_CATEGORY = { |
| "instruction_override": "hybrid", |
| "prompt_leakage": "rule-based", |
| "retrieved_indirect": "hybrid", |
| "evasion_obfuscation": "rule-based", |
| "adaptive_multiturn": "hybrid", |
| "benign_control": "hybrid", |
| } |
|
|
|
|
| def _map_category(case: dict[str, Any]) -> tuple[str, str, str]: |
| family = str(case.get("attack_family", "")).lower() |
| subtype = str(case.get("attack_subtype", "")).lower() |
| surface = str(case.get("attack_surface", "")).lower() |
| label = str(case.get("label", "")).lower() |
|
|
| if family in MAPPING_TABLE: |
| return MAPPING_TABLE[family] |
| if subtype == "hijacking": |
| return ("instruction_override", "s1_instruction_override", "high") |
| if subtype == "extraction": |
| return ("prompt_leakage", "s2_prompt_leakage", "high") |
| if surface == "tool_mediated": |
| return ("retrieved_indirect", "s3_retrieved_indirect", "medium") |
| if label == "prompt_leakage": |
| return ("prompt_leakage", "s2_prompt_leakage", "medium") |
| if label == "instruction_override": |
| return ("instruction_override", "s1_instruction_override", "medium") |
| return ("instruction_override", "s1_instruction_override", "low") |
|
|
|
|
| def _map_numeric_severity(case: dict[str, Any], confidence: str) -> tuple[int | None, str]: |
| severity = case.get("severity") |
| if isinstance(severity, int): |
| return severity, confidence |
|
|
| severity_level = case.get("severity_level") |
| if isinstance(severity_level, str): |
| mapped = SEVERITY_MAP.get(severity_level.lower()) |
| if mapped is not None: |
| return mapped, confidence |
| return None, "low" if confidence != "low" else confidence |
|
|
|
|
| def map_case(case: dict[str, Any]) -> dict[str, Any]: |
| enriched = dict(case) |
| category, success_definition_id, confidence = _map_category(case) |
| severity, confidence = _map_numeric_severity(case, confidence) |
|
|
| enriched["attack_category"] = category |
| enriched["success_definition_id"] = success_definition_id |
| enriched["mapping_confidence"] = confidence |
| enriched["severity"] = severity |
| enriched["evaluation_mode"] = EVALUATION_MODE_BY_CATEGORY.get(category, "hybrid") |
|
|
| provenance = dict(enriched.get("provenance") or {}) |
| if "source_file" not in provenance and enriched.get("source_reference"): |
| provenance["source_file"] = enriched["source_reference"] |
| if "dataset_split" not in provenance and enriched.get("benchmark_split"): |
| provenance["dataset_split"] = enriched["benchmark_split"] |
| if provenance: |
| enriched["provenance"] = provenance |
|
|
| if confidence in {"low", "medium"}: |
| enriched["mapping_audit"] = { |
| "source_field_used": ( |
| enriched.get("attack_family") |
| or enriched.get("attack_surface") |
| or enriched.get("label") |
| or "" |
| ), |
| "notes": "auto-mapped, review recommended", |
| } |
|
|
| return enriched |
|
|
|
|
| def summarize_mappings(cases: list[dict[str, Any]]) -> dict[str, dict[str, int]]: |
| categories = Counter() |
| confidences = Counter() |
| sources = Counter() |
|
|
| for case in cases: |
| enriched = map_case(case) |
| categories[enriched["attack_category"]] += 1 |
| confidences[enriched["mapping_confidence"]] += 1 |
| sources[str(enriched.get("source_name", "unknown"))] += 1 |
|
|
| return { |
| "attack_categories": dict(categories), |
| "mapping_confidence": dict(confidences), |
| "source_names": dict(sources), |
| } |
|
|
