Newer version of Presidio Streamlit

presidio_streamlit.py

@@ -1,4 +1,12 @@
-"""Streamlit app for Presidio."""
+"""Streamlit app for SolidPrivacy Scrub / Microsoft Presidio.
+
+Phase 1-3 update:
+- Dutch Legal Strict recognition profile;
+- Dutch legal test examples;
+- legal-aware replacement labels and scrub report download;
+- keeps current workflow: upload -> detect -> editable replacement table -> export.
+"""
+
 import logging
 import os
 import traceback
@@ -19,7 +27,6 @@ from presidio_helpers import (
     create_fake_data,
     analyzer_engine,
 )
-
 from document_tools import (
     uploaded_file_to_text,
     build_placeholder_replacements,
@@ -28,8 +35,8 @@ from document_tools import (
     docx_from_text,
     pdf_from_text,
     replacement_report_csv,
+    scrub_report_txt,
 )
-
 from replacement_memory import (
     load_remembered_replacements,
     save_remembered_replacements,
@@ -38,42 +45,61 @@ from replacement_memory import (
 )
 
 try:
-    from dutch_recognizers import get_dutch_entity_names
-except Exception:  # keep app usable while the new file is being added
-    def get_dutch_entity_names():
+    from dutch_recognizers import (
+        get_dutch_entity_names,
+        get_dutch_general_entity_names,
+        get_dutch_legal_entity_names,
+    )
+except Exception:  # keep app usable while new file is being added
+    def get_dutch_entity_names(include_legal=True):
+        return []
+
+    def get_dutch_general_entity_names():
+        return []
+
+    def get_dutch_legal_entity_names():
+        return []
+
+try:
+    from legal_test_examples import TEST_CASES, get_example_names, get_example_text
+except Exception:
+    TEST_CASES = []
+
+    def get_example_names():
         return []
 
+    def get_example_text(name: str):
+        return ""
+
+
 st.set_page_config(
-    page_title="Presidio demo",
+    page_title="SolidPrivacy Scrub",
     layout="wide",
     initial_sidebar_state="expanded",
-    menu_items={
-        "About": "https://microsoft.github.io/presidio/",
-    },
+    menu_items={"About": "https://microsoft.github.io/presidio/"},
 )
 
 dotenv.load_dotenv()
 logger = logging.getLogger("presidio-streamlit")
-
-
 allow_other_models = os.getenv("ALLOW_OTHER_MODELS", False)
 
 
 # Sidebar
 st.sidebar.header(
     """
-PII De-Identification with [Microsoft Presidio](https://microsoft.github.io/presidio/)
-"""
+    SolidPrivacy Scrub
+    
+    PII De-Identification with [Microsoft Presidio](https://microsoft.github.io/presidio/)
+    """
 )
 
-
 model_help_text = """
-    Select which Named Entity Recognition (NER) model to use for PII detection, in parallel to rule-based recognizers.
-    Presidio supports multiple NER packages off-the-shelf, such as spaCy, Huggingface, Stanza and Flair,
-    as well as service such as Azure Text Analytics PII.
-    """
-st_ta_key = st_ta_endpoint = ""
+Select which Named Entity Recognition (NER) model to use for PII detection,
+in parallel to rule-based recognizers. The Dutch Legal Strict layer is rule-based
+and does not require a cloud model.
+"""
 
+st_ta_key = st_ta_endpoint = ""
 model_list = [
     "spaCy/en_core_web_lg",
     "flair/ner-english-large",
@@ -85,7 +111,7 @@ model_list = [
 ]
 if not allow_other_models:
     model_list.pop()
-# Select model
+
 st_model = st.sidebar.selectbox(
     "NER model package",
     model_list,
@@ -93,10 +119,7 @@ st_model = st.sidebar.selectbox(
     help=model_help_text,
 )
 
-# Extract model package.
 st_model_package = st_model.split("/")[0]
-
-# Remove package prefix (if needed)
 st_model = (
     st_model
     if st_model_package.lower() not in ("spacy", "stanza", "huggingface")
@@ -107,31 +130,31 @@ if st_model == "Other":
     st_model_package = st.sidebar.selectbox(
         "NER model OSS package", options=["spaCy", "stanza", "Flair", "HuggingFace"]
     )
-    st_model = st.sidebar.text_input(f"NER model name", value="")
+    st_model = st.sidebar.text_input("NER model name", value="")
 
 if st_model == "Azure AI Language":
     st_ta_key = st.sidebar.text_input(
-        f"Azure AI Language key", value=os.getenv("TA_KEY", ""), type="password"
+        "Azure AI Language key", value=os.getenv("TA_KEY", ""), type="password"
     )
     st_ta_endpoint = st.sidebar.text_input(
-        f"Azure AI Language endpoint",
+        "Azure AI Language endpoint",
         value=os.getenv("TA_ENDPOINT", default=""),
-        help="For more info: https://learn.microsoft.com/en-us/azure/cognitive-services/language-service/personally-identifiable-information/overview",  # noqa: E501
+        help="For more info: https://learn.microsoft.com/en-us/azure/cognitive-services/language-service/personally-identifiable-information/overview",
     )
 
-
-st.sidebar.warning("Note: Models might take some time to download. ")
-
+st.sidebar.warning("Note: some NER models might take time to download/load.")
 analyzer_params = (st_model_package, st_model, st_ta_key, st_ta_endpoint)
-logger.debug(f"analyzer_params: {analyzer_params}")
+logger.debug("analyzer_params: %s", analyzer_params)
 
 st_recognition_profile = st.sidebar.selectbox(
     "Recognition profile",
-    ["Dutch / EU", "General / International"],
+    ["Dutch Legal Strict", "Dutch / EU", "General / International"],
     index=0,
     help=(
-        "Dutch / EU enables Dutch pattern recognizers such as BSN, postcode, "
-        "KvK, BTW/VAT, Dutch IBAN, Dutch phone numbers and Dutch license plates."
+        "Dutch Legal Strict adds Dutch legal/matter identifiers such as zaaknummer, "
+        "rolnummer, parketnummer, dossiernummer, cliëntnummer, CJIB and ECLI. "
+        "Dutch / EU enables general Dutch identifiers such as BSN, postcode, KvK, BTW/VAT, "
+        "Dutch IBAN, Dutch phone numbers and Dutch address patterns."
     ),
 )
 
@@ -140,47 +163,44 @@ st_operator = st.sidebar.selectbox(
     ["redact", "replace", "synthesize", "highlight", "mask", "hash", "encrypt"],
     index=1,
     help="""
-    Select which manipulation to the text is requested after PII has been identified.\n
-    - Redact: Completely remove the PII text\n
-    - Replace: Replace the PII text with a constant, e.g. <PERSON>\n
-    - Synthesize: Replace with fake values (requires an OpenAI key)\n
-    - Highlight: Shows the original text with PII highlighted in colors\n
-    - Mask: Replaces a requested number of characters with an asterisk (or other mask character)\n
-    - Hash: Replaces with the hash of the PII string\n
-    - Encrypt: Replaces with an AES encryption of the PII string, allowing the process to be reversed
-         """,
+    Select which manipulation is requested after PII has been identified.
+    - Redact: completely remove the PII text
+    - Replace: replace PII with a placeholder
+    - Synthesize: replace with fake values; requires an OpenAI key
+    - Highlight: show original text with PII highlighted
+    - Mask: replace characters with a mask character
+    - Hash: replace with a hash
+    - Encrypt: replace with AES encryption, reversible with the key
+    """,
 )
+
 st_mask_char = "*"
 st_number_of_chars = 15
 st_encrypt_key = "WmZq4t7w!z%C&F)J"
-
 open_ai_params = None
-
-logger.debug(f"st_operator: {st_operator}")
+logger.debug("st_operator: %s", st_operator)
 
 
 def set_up_openai_synthesis():
     """Set up the OpenAI API key and model for text synthesis."""
-
     if os.getenv("OPENAI_TYPE", default="openai") == "Azure":
         openai_api_type = "azure"
         st_openai_api_base = st.sidebar.text_input(
-            "Azure OpenAI base URL",
-            value=os.getenv("AZURE_OPENAI_ENDPOINT", default=""),
+            "Azure OpenAI base URL", value=os.getenv("AZURE_OPENAI_ENDPOINT", default="")
         )
         openai_key = os.getenv("AZURE_OPENAI_KEY", default="")
         st_deployment_id = st.sidebar.text_input(
             "Deployment name", value=os.getenv("AZURE_OPENAI_DEPLOYMENT", default="")
         )
         st_openai_version = st.sidebar.text_input(
-            "OpenAI version",
-            value=os.getenv("OPENAI_API_VERSION", default="2023-05-15"),
+            "OpenAI version", value=os.getenv("OPENAI_API_VERSION", default="2023-05-15")
         )
     else:
         openai_api_type = "openai"
         st_openai_version = st_openai_api_base = None
         st_deployment_id = ""
         openai_key = os.getenv("OPENAI_KEY", default="")
+
     st_openai_key = st.sidebar.text_input(
         "OPENAI_KEY",
         value=openai_key,
@@ -206,9 +226,7 @@ if st_operator == "mask":
     st_number_of_chars = st.sidebar.number_input(
         "number of chars", value=st_number_of_chars, min_value=0, max_value=100
     )
-    st_mask_char = st.sidebar.text_input(
-        "Mask character", value=st_mask_char, max_chars=1
-    )
+    st_mask_char = st.sidebar.text_input("Mask character", value=st_mask_char, max_chars=1)
 elif st_operator == "encrypt":
     st_encrypt_key = st.sidebar.text_input("AES key", value=st_encrypt_key)
 elif st_operator == "synthesize":
@@ -220,7 +238,6 @@ elif st_operator == "synthesize":
         st_openai_key,
         st_openai_model,
     ) = set_up_openai_synthesis()
-
     open_ai_params = OpenAIParams(
         openai_key=st_openai_key,
         model=st_openai_model,
@@ -230,91 +247,73 @@ elif st_operator == "synthesize":
         api_type=openai_api_type,
     )
 
+st_threshold_default = 0.30 if st_recognition_profile == "Dutch Legal Strict" else 0.35
 st_threshold = st.sidebar.slider(
     label="Acceptance threshold",
     min_value=0.0,
     max_value=1.0,
-    value=0.35,
-    help="Define the threshold for accepting a detection as PII. See more here: ",
+    value=st_threshold_default,
+    help="Define the threshold for accepting a detection as PII.",
 )
 
 st_return_decision_process = st.sidebar.checkbox(
     "Add analysis explanations to findings",
     value=False,
-    help="Add the decision process to the output table. "
-    "More information can be found here: https://microsoft.github.io/presidio/analyzer/decision_process/",
-)
-
-# Allow and deny lists
-st_deny_allow_expander = st.sidebar.expander(
-    "Allowlists and denylists",
-    expanded=False,
+    help=(
+        "Add the decision process to the output table. More information: "
+        "https://microsoft.github.io/presidio/analyzer/decision_process/"
+    ),
 )
 
+st_deny_allow_expander = st.sidebar.expander("Allowlists and denylists", expanded=False)
 with st_deny_allow_expander:
-    st_allow_list = st_tags(
-        label="Add words to the allowlist", text="Enter word and press enter."
-    )
-    st.caption(
-        "Allowlists contain words that are not considered PII, but are detected as such."
-    )
+    st_allow_list = st_tags(label="Add words to the allowlist", text="Enter word and press enter.")
+    st.caption("Allowlists contain words that are not considered PII, but are detected as such.")
+    st_deny_list = st_tags(label="Add words to the denylist", text="Enter word and press enter.")
+    st.caption("Denylists contain words that are considered PII, but are not detected as such.")
 
-    st_deny_list = st_tags(
-        label="Add words to the denylist", text="Enter word and press enter."
-    )
-    st.caption(
-        "Denylists contain words that are considered PII, but are not detected as such."
-    )
-# Main panel
 
+# Main panel
 with st.expander("About this demo", expanded=False):
     st.info(
-        """Presidio is an open source customizable framework for PII detection and de-identification.
-        \n\n[Code](https://aka.ms/presidio) | 
-        [Tutorial](https://microsoft.github.io/presidio/tutorial/) | 
-        [Installation](https://microsoft.github.io/presidio/installation/) | 
+        """
+        Presidio is an open source customizable framework for PII detection and de-identification.
+
+        [Code](https://aka.ms/presidio) | [Tutorial](https://microsoft.github.io/presidio/tutorial/) |
+        [Installation](https://microsoft.github.io/presidio/installation/) |
         [FAQ](https://microsoft.github.io/presidio/faq/) |
-        [Feedback](https://forms.office.com/r/9ufyYjfDaY) |"""
+        [Feedback](https://forms.office.com/r/9ufyYjfDaY)
+        """
     )
-
     st.info(
         """
-    Use this demo to:
-    - Experiment with different off-the-shelf models and NLP packages.
-    - Explore the different de-identification options, including redaction, masking, encryption and more.
-    - Generate synthetic text with Microsoft Presidio and OpenAI.
-    - Configure allow and deny lists.
-    
-    This demo website shows some of Presidio's capabilities.
-    [Visit our website](https://microsoft.github.io/presidio) for more info,
-    samples and deployment options.    
-    """
-    )
-
-    st.markdown(
-        "[![Pypi Downloads](https://img.shields.io/pypi/dm/presidio-analyzer.svg)](https://img.shields.io/pypi/dm/presidio-analyzer.svg)"  # noqa
-        "[![MIT license](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://opensource.org/licenses/MIT)"
-        "![GitHub Repo stars](https://img.shields.io/github/stars/microsoft/presidio?style=social)"
+        SolidPrivacy Scrub extends the demo with Dutch/EU and Dutch legal recognizers.
+        For legal/confidential material, use fake documents in this public Space.
+        The recognizer pack is designed to be local/offline compatible for a future desktop/MSI version.
+        """
     )
 
 analyzer_load_state = st.info("Starting Presidio analyzer...")
-
 analyzer_load_state.empty()
 
-if st_recognition_profile == "Dutch / EU":
+if st_recognition_profile == "Dutch Legal Strict":
+    st.info(
+        "Dutch Legal Strict mode is active. The app adds Dutch/EU recognizers plus legal/matter identifiers: "
+        "zaaknummer, rolnummer, rekestnummer, parketnummer, dossiernummer, cliëntnummer, CJIB, ECLI, "
+        "legal party references and court/authority references. Always review the editable replacement table."
+    )
+elif st_recognition_profile == "Dutch / EU":
     st.info(
-        "Dutch / EU mode is active. The app adds Dutch pattern recognizers "
-        "for BSN, postcode, KvK, BTW/VAT, Dutch IBAN, Dutch phone numbers, "
-        "license plates, rijbewijs-style numbers and BIG numbers. Always review "
-        "the editable replacement table before exporting."
+        "Dutch / EU mode is active. The app adds Dutch pattern recognizers for BSN, postcode, KvK, BTW/VAT, "
+        "Dutch IBAN, Dutch phone numbers, addresses, license plates, rijbewijs-style numbers and BIG numbers. "
+        "Always review the editable replacement table before exporting."
     )
 
 # Read default text
-with open("demo_text.txt") as f:
+with open("demo_text.txt", encoding="utf-8") as f:
     demo_text = f.readlines()
 
 st.subheader("Document input")
-
 uploaded_file = st.file_uploader(
     "Upload a .txt, .docx, or text-based .pdf file",
     type=["txt", "docx", "pdf"],
@@ -324,6 +323,17 @@ uploaded_file = st.file_uploader(
 uploaded_file_type = None
 input_text = "".join(demo_text)
 
+if st_recognition_profile == "Dutch Legal Strict" and TEST_CASES:
+    with st.expander("Use a fake Dutch legal test example", expanded=False):
+        sample_name = st.selectbox(
+            "Load synthetic legal example",
+            ["Do not load a test example"] + get_example_names(),
+            index=0,
+        )
+        if sample_name != "Do not load a test example" and uploaded_file is None:
+            input_text = get_example_text(sample_name)
+            st.caption("Loaded synthetic example text. No real personal data is included.")
+
 if uploaded_file is not None:
     try:
         input_text, uploaded_file_type = uploaded_file_to_text(uploaded_file)
@@ -331,12 +341,8 @@ if uploaded_file is not None:
     except Exception as upload_error:
         st.error(f"Could not read uploaded file: {upload_error}")
 
-# Create two columns for before and after
 col1, col2 = st.columns(2)
-
-# Before:
 col1.subheader("Input")
-
 st_text = col1.text_area(
     label="Enter text or review extracted document text",
     value=input_text,
@@ -345,46 +351,51 @@ st_text = col1.text_area(
 )
 
 try:
-    # Choose entities
     all_supported_entities = list(get_supported_entities(*analyzer_params))
-    dutch_entities = set(get_dutch_entity_names())
-
-    if st_recognition_profile == "Dutch / EU":
-        preferred_entities = {
-            "PERSON",
-            "LOCATION",
-            "ORGANIZATION",
-            "EMAIL_ADDRESS",
-            "PHONE_NUMBER",
-            "IBAN_CODE",
-            "URL",
-            "IP_ADDRESS",
-            "GENERIC_PII",
-        } | dutch_entities
-        default_entities = [
-            entity for entity in all_supported_entities if entity in preferred_entities
-        ]
+    general_dutch_entities = set(get_dutch_general_entity_names())
+    legal_dutch_entities = set(get_dutch_legal_entity_names())
+    all_dutch_entities = set(get_dutch_entity_names(include_legal=True))
+
+    base_preferred_entities = {
+        "PERSON",
+        "LOCATION",
+        "ORGANIZATION",
+        "EMAIL_ADDRESS",
+        "PHONE_NUMBER",
+        "IBAN_CODE",
+        "URL",
+        "IP_ADDRESS",
+        "GENERIC_PII",
+        "DATE_TIME",
+    }
+
+    if st_recognition_profile == "Dutch Legal Strict":
+        preferred_entities = base_preferred_entities | all_dutch_entities
+    elif st_recognition_profile == "Dutch / EU":
+        preferred_entities = base_preferred_entities | general_dutch_entities
     else:
-        default_entities = list(all_supported_entities)
+        preferred_entities = set(all_supported_entities)
+
+    default_entities = [entity for entity in all_supported_entities if entity in preferred_entities]
 
     st_entities_expander = st.sidebar.expander("Choose entities to look for")
     st_entities = st_entities_expander.multiselect(
         label="Which entities to look for?",
         options=all_supported_entities,
         default=default_entities,
-        help="Limit the list of PII entities detected. "
-        "Dutch / EU mode adds recognizers such as NL_BSN, NL_POSTCODE, "
-        "NL_KVK_NUMBER, NL_VAT_NUMBER, NL_IBAN and NL_PHONE_NUMBER.",
+        help=(
+            "Dutch / EU mode adds recognizers such as NL_BSN, NL_POSTCODE, NL_KVK_NUMBER, "
+            "NL_VAT_NUMBER, NL_IBAN and NL_PHONE_NUMBER. Dutch Legal Strict additionally adds "
+            "NL_ECLI, NL_LEGAL_CASE_NUMBER, NL_PARKETNUMMER, NL_DOSSIER_NUMBER, NL_CLIENT_NUMBER and related legal IDs."
+        ),
     )
 
-    # Before
     analyzer_load_state = st.info("Starting Presidio analyzer...")
     analyzer = analyzer_engine(*analyzer_params)
     analyzer_load_state.empty()
 
-    # The current demo uses English NER models. Dutch/EU pattern recognizers
-    # are registered under language="en" so they can run without requiring a
-    # separate Dutch NLP model.
+    # The current demo uses English NER models. Dutch/EU pattern recognizers are
+    # registered under language="en" so they can run without a separate Dutch NLP model.
     st_analyze_results = analyze(
         *analyzer_params,
         text=st_text,
@@ -396,10 +407,9 @@ try:
         deny_list=st_deny_list,
     )
 
-    # After
     if st_operator not in ("highlight", "synthesize"):
         with col2:
-            st.subheader(f"Output")
+            st.subheader("Output")
             st_anonymize_results = anonymize(
                 text=st_text,
                 operator=st_operator,
@@ -408,37 +418,26 @@ try:
                 encrypt_key=st_encrypt_key,
                 analyze_results=st_analyze_results,
             )
-            st.text_area(
-                label="De-identified", value=st_anonymize_results.text, height=400
-            )
-        # Build stable placeholder suggestions from Presidio
-        replacements, report_rows = build_placeholder_replacements(
-            st_text,
-            st_analyze_results,
-        )
+            st.text_area(label="De-identified", value=st_anonymize_results.text, height=400)
+
+        _, report_rows = build_placeholder_replacements(st_text, st_analyze_results)
 
         st.divider()
         st.subheader("Review replacement table before export")
-
         st.caption(
             "Untick false positives, change placeholders, add your own word pairs, "
             "and tick Remember for pairs you want to reuse in future documents."
         )
 
-        # Build editable table rows from remembered replacements + Presidio suggestions
         remembered_rows = load_remembered_replacements()
-
         default_editor_rows = []
         seen_find_values = set()
 
-        # First load remembered pairs
         for row in remembered_rows:
             find_text = str(row.get("find", "")).strip()
             replace_with = str(row.get("replace_with", "")).strip()
-
             if not find_text or not replace_with:
                 continue
-
             default_editor_rows.append(
                 {
                     "include": row.get("include", True),
@@ -451,16 +450,10 @@ try:
             )
             seen_find_values.add(find_text)
 
-        # Then add Presidio suggestions, unless already covered by a remembered pair
         for row in report_rows:
             find_text = str(row.get("detected_text", "")).strip()
-
-            if not find_text:
-                continue
-
-            if find_text in seen_find_values:
+            if not find_text or find_text in seen_find_values:
                 continue
-
             default_editor_rows.append(
                 {
                     "include": True,
@@ -472,7 +465,6 @@ try:
                 }
             )
 
-        # If nothing was detected and nothing was remembered, still show an empty editable row
         if not default_editor_rows:
             default_editor_rows = [
                 {
@@ -486,7 +478,6 @@ try:
             ]
 
         replacement_editor_df = pd.DataFrame(default_editor_rows)
-
         edited_replacements_df = st.data_editor(
             replacement_editor_df,
             hide_index=True,
@@ -495,31 +486,22 @@ try:
             column_order=["include", "remember", "find", "replace_with", "entity_type", "score"],
             column_config={
                 "include": st.column_config.CheckboxColumn(
-                    "Use",
-                    help="Untick to exclude this replacement from the export.",
-                    default=True,
+                    "Use", help="Untick to exclude this replacement from the export.", default=True
                 ),
                 "remember": st.column_config.CheckboxColumn(
-                    "Remember",
-                    help="Save this replacement pair for future documents/sessions.",
-                    default=False,
+                    "Remember", help="Save this replacement pair for future documents/sessions.", default=False
                 ),
                 "find": st.column_config.TextColumn(
-                    "Find text",
-                    help="The exact text that should be replaced.",
+                    "Find text", help="The exact text that should be replaced."
                 ),
                 "replace_with": st.column_config.TextColumn(
-                    "Replace with",
-                    help="The placeholder to insert.",
+                    "Replace with", help="The placeholder to insert."
                 ),
                 "entity_type": st.column_config.TextColumn(
-                    "Entity type",
-                    help="Presidio entity type or MANUAL.",
+                    "Entity type", help="Presidio entity type or MANUAL."
                 ),
                 "score": st.column_config.NumberColumn(
-                    "Score",
-                    help="Presidio confidence score, if available.",
-                    format="%.3f",
+                    "Score", help="Presidio confidence score, if available.", format="%.3f"
                 ),
             },
             key="replacement_editor",
@@ -549,25 +531,17 @@ try:
                 return bool(value)
             return str(value).strip().lower() in ("true", "1", "yes", "y", "checked")
 
-        # Build final replacements from edited table
         edited_replacements = {}
         edited_report_rows = []
-
         for _, row in edited_replacements_df.iterrows():
             include = safe_bool(row.get("include", False))
             find_text = safe_cell(row.get("find", ""))
             replace_text = safe_cell(row.get("replace_with", ""))
             entity_type = safe_cell(row.get("entity_type", "MANUAL")) or "MANUAL"
             score = row.get("score", None)
-
-            if not include:
-                continue
-
-            if not find_text or not replace_text:
+            if not include or not find_text or not replace_text:
                 continue
-
             edited_replacements[find_text] = replace_text
-
             edited_report_rows.append(
                 {
                     "entity_type": entity_type,
@@ -579,58 +553,40 @@ try:
 
         st.info(f"{len(edited_replacements)} replacement pair(s) will be applied to the exports.")
 
-        # Apply edited replacements
         export_text = apply_replacements_to_text(st_text, edited_replacements)
-
         with st.expander("Preview anonymized text generated from edited table", expanded=False):
-            st.text_area(
-                label="Preview",
-                value=export_text,
-                height=300,
-                key="edited_export_preview",
-            )
+            st.text_area(label="Preview", value=export_text, height=300, key="edited_export_preview")
 
         st.subheader("Remember reusable replacements")
-
         remember_rows_to_save = []
-
         for _, row in edited_replacements_df.iterrows():
             include = safe_bool(row.get("include", False))
             remember = safe_bool(row.get("remember", False))
             find_text = safe_cell(row.get("find", ""))
             replace_text = safe_cell(row.get("replace_with", ""))
             entity_type = safe_cell(row.get("entity_type", "REMEMBERED")) or "REMEMBERED"
-
             if include and remember and find_text and replace_text:
                 remember_rows_to_save.append(
-                    {
-                        "find": find_text,
-                        "replace_with": replace_text,
-                        "entity_type": entity_type,
-                    }
+                    {"find": find_text, "replace_with": replace_text, "entity_type": entity_type}
                 )
 
         memory_col1, memory_col2 = st.columns(2)
-
         with memory_col1:
             if st.button("Save remembered replacements"):
                 saved_count = save_remembered_replacements(remember_rows_to_save)
                 st.success(f"Saved {saved_count} remembered replacement pair(s).")
                 st.info(f"Memory file: {get_memory_file_path()}")
-
         with memory_col2:
             if st.button("Clear remembered replacements"):
                 clear_remembered_replacements()
                 st.warning("Remembered replacements cleared.")
 
         st.subheader("Export anonymized files")
-
         if uploaded_file is not None:
             st.info(f"Uploaded file detected for export: {uploaded_file.name}")
         else:
             st.info("No uploaded file detected for export. Exporting from text area only.")
 
-        # TXT export
         st.download_button(
             label="Download anonymized text (.txt)",
             data=export_text.encode("utf-8"),
@@ -638,8 +594,6 @@ try:
             mime="text/plain",
             key="download_txt",
         )
-
-        # CSV replacement report / reusable mapping
         st.download_button(
             label="Download replacement table (.csv)",
             data=replacement_report_csv(edited_report_rows),
@@ -647,8 +601,18 @@ try:
             mime="text/csv",
             key="download_csv",
         )
+        st.download_button(
+            label="Download scrub report (.txt)",
+            data=scrub_report_txt(
+                edited_report_rows,
+                profile=st_recognition_profile,
+                source_filename=uploaded_file.name if uploaded_file is not None else None,
+            ),
+            file_name="scrub_report.txt",
+            mime="text/plain",
+            key="download_scrub_report",
+        )
 
-        # DOCX export
         try:
             if uploaded_file is not None and uploaded_file.name.lower().endswith(".docx"):
                 docx_bytes = anonymized_docx_from_original(uploaded_file, edited_replacements)
@@ -656,7 +620,6 @@ try:
             else:
                 docx_bytes = docx_from_text(export_text)
                 docx_filename = "anonymized_text.docx"
-
             st.download_button(
                 label="Download anonymized Word file (.docx)",
                 data=docx_bytes,
@@ -664,11 +627,9 @@ try:
                 mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
                 key="download_docx",
             )
-
         except Exception as docx_error:
             st.error(f"Could not create DOCX export: {docx_error}")
 
-        # PDF export
         try:
             st.download_button(
                 label="Download anonymized PDF (.pdf)",
@@ -677,35 +638,23 @@ try:
                 mime="application/pdf",
                 key="download_pdf",
             )
-
         except Exception as pdf_error:
             st.error(f"Could not create PDF export: {pdf_error}")
 
     elif st_operator == "synthesize":
         with col2:
-            st.subheader(f"OpenAI Generated output")
-            fake_data = create_fake_data(
-                st_text,
-                st_analyze_results,
-                open_ai_params,
-            )
+            st.subheader("OpenAI Generated output")
+            fake_data = create_fake_data(st_text, st_analyze_results, open_ai_params)
             st.text_area(label="Synthetic data", value=fake_data, height=400)
     else:
         st.subheader("Highlighted")
         annotated_tokens = annotate(text=st_text, analyze_results=st_analyze_results)
-        # annotated_tokens
         annotated_text(*annotated_tokens)
 
-    # table result
-    st.subheader(
-        "Findings"
-        if not st_return_decision_process
-        else "Findings with decision factors"
-    )
+    st.subheader("Findings" if not st_return_decision_process else "Findings with decision factors")
     if st_analyze_results:
         df = pd.DataFrame.from_records([r.to_dict() for r in st_analyze_results])
         df["text"] = [st_text[res.start : res.end] for res in st_analyze_results]
-
         df_subset = df[["entity_type", "text", "start", "end", "score"]].rename(
             {
                 "entity_type": "Entity type",
@@ -716,7 +665,6 @@ try:
             },
             axis=1,
         )
-        df_subset["Text"] = [st_text[res.start : res.end] for res in st_analyze_results]
         if st_return_decision_process:
             analysis_explanation_df = pd.DataFrame.from_records(
                 [r.analysis_explanation.to_dict() for r in st_analyze_results]
@@ -731,15 +679,4 @@ except Exception as e:
     traceback.print_exc()
     st.error(e)
 
-components.html(
-    """
-    <script type="text/javascript">
-    (function(c,l,a,r,i,t,y){
-        c[a]=c[a]||function(){(c[a].q=c[a].q||[]).push(arguments)};
-        t=l.createElement(r);t.async=1;t.src="https://www.clarity.ms/tag/"+i;
-        y=l.getElementsByTagName(r)[0];y.parentNode.insertBefore(t,y);
-    })(window, document, "clarity", "script", "h7f8bp42n8");
-    </script>
-    """
-)
- 
+components.html(""" """)