solidprivacy-nl commited on
Commit
e322da3
·
1 Parent(s): d76276b

Implement Dutch Legal UI layer

Browse files
Files changed (1) hide show
  1. presidio_streamlit.py +329 -336
presidio_streamlit.py CHANGED
@@ -1,18 +1,14 @@
1
- """Streamlit app for SolidPrivacy Scrub / Microsoft Presidio.
2
-
3
- Phase 1-3 v8 update:
4
- - adds an Audit & Candidate Review layer for suspicious values not automatically masked;
5
- - keeps candidate values unchecked by default, so the user decides whether to include them;
6
- - supports a structured observation workflow instead of one-off after-the-fact patches.
7
-
8
- Phase 1-3 update:
9
- - Dutch Legal Strict recognition profile;
10
- - Dutch legal test examples;
11
- - Dutch Legal Reference Taxonomy for context-based reference codes;
12
- - legal-aware replacement labels and scrub report download;
13
- - keeps current workflow: upload -> detect -> editable replacement table -> export.
14
  """
15
 
 
 
16
  import ast
17
  import logging
18
  import os
@@ -51,6 +47,18 @@ from replacement_memory import (
51
  clear_remembered_replacements,
52
  get_memory_file_path,
53
  )
 
 
 
 
 
 
 
 
 
 
 
 
54
 
55
  try:
56
  from candidate_scanner import scan_unmasked_candidates
@@ -64,7 +72,7 @@ try:
64
  get_dutch_general_entity_names,
65
  get_dutch_legal_entity_names,
66
  )
67
- except Exception: # keep app usable while new file is being added
68
  def get_dutch_entity_names(include_legal=True):
69
  return []
70
 
@@ -74,10 +82,11 @@ except Exception: # keep app usable while new file is being added
74
  def get_dutch_legal_entity_names():
75
  return []
76
 
 
77
  LEGAL_EXAMPLES_IMPORT_ERROR = None
78
 
79
  EMBEDDED_LEGAL_TEST_CASES = {
80
- "Fallback - referenties en administratieve nummers": """Cliëntnummer: CL-FAM-55201.
81
  De schoolreferentie is HRZ-SAM-2026-04.
82
  In het verslag van Stichting Horizonzorg wordt dezelfde referentie HRZ-SAM-2026-04 genoemd.
83
  De factuur met nummer FACT-2026-4481 is onbetaald gebleven.
@@ -85,7 +94,7 @@ De interne klantreferentie van eiser is WR-KLANT-2026-7712.
85
  De zaakreferentie is ZK-WOON-55091.
86
  Het artikel 7:669 BW mag niet worden gemaskeerd.
87
  De datum 15-12-2026 mag niet als referentie worden gezien.
88
- Het bedrag 1.250,00 mag niet als referentie worden gezien.
89
  """,
90
  "Fallback - familierecht contextbehoud": """Aan de Rechtbank Amsterdam
91
 
@@ -98,15 +107,17 @@ Verweerder Peter Bakker woont aan Laan van Meerdervoort 55, 2517 AM Den Haag.
98
  """,
99
  }
100
 
 
 
 
 
 
 
 
101
 
102
- def _load_legal_test_cases_from_file():
103
- """Load legal examples as data instead of importing the module.
104
 
105
- Streamlit can re-run scripts while modules are still initializing. A normal
106
- `from legal_test_examples import TEST_CASES` can therefore fail with a
107
- misleading circular-import warning. The examples file is pure data, so we
108
- parse the TEST_CASES literal directly from disk and avoid executing imports.
109
- """
110
  examples_path = Path(__file__).with_name("legal_test_examples.py")
111
  if not examples_path.exists():
112
  raise FileNotFoundError(f"{examples_path} does not exist")
@@ -122,8 +133,7 @@ def _load_legal_test_cases_from_file():
122
  and node.target.id == "TEST_CASES"
123
  )
124
  if is_test_cases:
125
- value = node.value
126
- cases = ast.literal_eval(value)
127
  if not isinstance(cases, list):
128
  raise ValueError("TEST_CASES is not a list")
129
  return cases
@@ -139,7 +149,7 @@ except Exception as exc:
139
 
140
  def get_example_names():
141
  if TEST_CASES:
142
- return [str(case.get("name", "Unnamed example")) for case in TEST_CASES]
143
  return list(EMBEDDED_LEGAL_TEST_CASES.keys())
144
 
145
 
@@ -150,117 +160,141 @@ def get_example_text(name: str):
150
  return EMBEDDED_LEGAL_TEST_CASES.get(name, "")
151
 
152
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  st.set_page_config(
154
- page_title="SolidPrivacy Scrub",
155
  layout="wide",
156
  initial_sidebar_state="expanded",
157
- menu_items={"About": "https://microsoft.github.io/presidio/"},
158
  )
159
 
160
  dotenv.load_dotenv()
161
- logger = logging.getLogger("presidio-streamlit")
162
  allow_other_models = os.getenv("ALLOW_OTHER_MODELS", False)
163
 
 
 
164
 
165
- # Sidebar
166
- st.sidebar.header(
167
- """
168
- SolidPrivacy Scrub
169
-
170
- PII De-Identification with [Microsoft Presidio](https://microsoft.github.io/presidio/)
171
- """
172
  )
173
-
174
- model_help_text = """
175
- Select which Named Entity Recognition (NER) model to use for PII detection,
176
- in parallel to rule-based recognizers. The Dutch Legal Strict layer is rule-based
177
- and does not require a cloud model.
178
- """
179
-
180
- st_ta_key = st_ta_endpoint = ""
181
- model_list = [
182
- "spaCy/en_core_web_lg",
183
- "flair/ner-english-large",
184
- "HuggingFace/obi/deid_roberta_i2b2",
185
- "HuggingFace/StanfordAIMI/stanford-deidentifier-base",
186
- "stanza/en",
187
- "Azure AI Language",
188
- "Other",
189
- ]
190
- if not allow_other_models:
191
- model_list.pop()
192
-
193
- st_model = st.sidebar.selectbox(
194
- "NER model package",
195
- model_list,
196
- index=1,
197
- help=model_help_text,
198
  )
 
199
 
200
- st_model_package = st_model.split("/")[0]
201
- st_model = (
202
- st_model
203
- if st_model_package.lower() not in ("spacy", "stanza", "huggingface")
204
- else "/".join(st_model.split("/")[1:])
205
- )
206
 
207
- if st_model == "Other":
208
- st_model_package = st.sidebar.selectbox(
209
- "NER model OSS package", options=["spaCy", "stanza", "Flair", "HuggingFace"]
 
 
210
  )
211
- st_model = st.sidebar.text_input("NER model name", value="")
212
-
213
- if st_model == "Azure AI Language":
214
- st_ta_key = st.sidebar.text_input(
215
- "Azure AI Language key", value=os.getenv("TA_KEY", ""), type="password"
 
 
 
 
 
 
 
 
 
 
 
 
 
216
  )
217
- st_ta_endpoint = st.sidebar.text_input(
218
- "Azure AI Language endpoint",
219
- value=os.getenv("TA_ENDPOINT", default=""),
220
- help="For more info: https://learn.microsoft.com/en-us/azure/cognitive-services/language-service/personally-identifiable-information/overview",
 
 
221
  )
222
 
223
- st.sidebar.warning("Note: some NER models might take time to download/load.")
224
- analyzer_params = (st_model_package, st_model, st_ta_key, st_ta_endpoint)
225
- logger.debug("analyzer_params: %s", analyzer_params)
 
 
226
 
227
- st_recognition_profile = st.sidebar.selectbox(
228
- "Recognition profile",
229
- ["Dutch Legal Strict", "Dutch / EU", "General / International"],
230
- index=0,
231
- help=(
232
- "Dutch Legal Strict adds Dutch legal/matter identifiers such as zaaknummer, "
233
- "rolnummer, parketnummer, dossiernummer, cliëntnummer, CJIB and ECLI. "
234
- "Dutch / EU enables general Dutch identifiers such as BSN, postcode, KvK, BTW/VAT, "
235
- "Dutch IBAN, Dutch phone numbers and Dutch address patterns."
236
- ),
237
- )
238
 
239
- st_operator = st.sidebar.selectbox(
240
- "De-identification approach",
241
- ["redact", "replace", "synthesize", "highlight", "mask", "hash", "encrypt"],
242
- index=1,
243
- help="""
244
- Select which manipulation is requested after PII has been identified.
245
- - Redact: completely remove the PII text
246
- - Replace: replace PII with a placeholder
247
- - Synthesize: replace with fake values; requires an OpenAI key
248
- - Highlight: show original text with PII highlighted
249
- - Mask: replace characters with a mask character
250
- - Hash: replace with a hash
251
- - Encrypt: replace with AES encryption, reversible with the key
252
- """,
253
- )
 
 
 
 
 
 
 
254
 
255
- st_mask_char = "*"
256
- st_number_of_chars = 15
257
- st_encrypt_key = "WmZq4t7w!z%C&F)J"
258
  open_ai_params = None
259
- logger.debug("st_operator: %s", st_operator)
260
 
261
 
262
  def set_up_openai_synthesis():
263
- """Set up the OpenAI API key and model for text synthesis."""
264
  if os.getenv("OPENAI_TYPE", default="openai") == "Azure":
265
  openai_api_type = "azure"
266
  st_openai_api_base = st.sidebar.text_input(
@@ -279,16 +313,10 @@ def set_up_openai_synthesis():
279
  st_deployment_id = ""
280
  openai_key = os.getenv("OPENAI_KEY", default="")
281
 
282
- st_openai_key = st.sidebar.text_input(
283
- "OPENAI_KEY",
284
- value=openai_key,
285
- help="See https://help.openai.com/en/articles/4936850-where-do-i-find-my-secret-api-key for more info.",
286
- type="password",
287
- )
288
  st_openai_model = st.sidebar.text_input(
289
- "OpenAI model for text synthesis",
290
  value=os.getenv("OPENAI_MODEL", default="gpt-3.5-turbo-instruct"),
291
- help="See more here: https://platform.openai.com/docs/models/",
292
  )
293
  return (
294
  openai_api_type,
@@ -300,14 +328,7 @@ def set_up_openai_synthesis():
300
  )
301
 
302
 
303
- if st_operator == "mask":
304
- st_number_of_chars = st.sidebar.number_input(
305
- "number of chars", value=st_number_of_chars, min_value=0, max_value=100
306
- )
307
- st_mask_char = st.sidebar.text_input("Mask character", value=st_mask_char, max_chars=1)
308
- elif st_operator == "encrypt":
309
- st_encrypt_key = st.sidebar.text_input("AES key", value=st_encrypt_key)
310
- elif st_operator == "synthesize":
311
  (
312
  openai_api_type,
313
  st_openai_api_base,
@@ -325,122 +346,85 @@ elif st_operator == "synthesize":
325
  api_type=openai_api_type,
326
  )
327
 
328
- st_threshold_default = 0.30 if st_recognition_profile == "Dutch Legal Strict" else 0.35
329
- st_threshold = st.sidebar.slider(
330
- label="Acceptance threshold",
331
- min_value=0.0,
332
- max_value=1.0,
333
- value=st_threshold_default,
334
- help="Define the threshold for accepting a detection as PII.",
335
- )
336
-
337
- st_return_decision_process = st.sidebar.checkbox(
338
- "Add analysis explanations to findings",
339
- value=False,
340
- help=(
341
- "Add the decision process to the output table. More information: "
342
- "https://microsoft.github.io/presidio/analyzer/decision_process/"
343
- ),
344
- )
345
-
346
- st_deny_allow_expander = st.sidebar.expander("Allowlists and denylists", expanded=False)
347
- with st_deny_allow_expander:
348
- st_allow_list = st_tags(label="Add words to the allowlist", text="Enter word and press enter.")
349
- st.caption("Allowlists contain words that are not considered PII, but are detected as such.")
350
- st_deny_list = st_tags(label="Add words to the denylist", text="Enter word and press enter.")
351
- st.caption("Denylists contain words that are considered PII, but are not detected as such.")
352
-
353
 
354
- # Main panel
355
- with st.expander("About this demo", expanded=False):
356
- st.info(
357
- """
358
- Presidio is an open source customizable framework for PII detection and de-identification.
359
-
360
- [Code](https://aka.ms/presidio) | [Tutorial](https://microsoft.github.io/presidio/tutorial/) |
361
- [Installation](https://microsoft.github.io/presidio/installation/) |
362
- [FAQ](https://microsoft.github.io/presidio/faq/) |
363
- [Feedback](https://forms.office.com/r/9ufyYjfDaY)
364
- """
365
  )
366
- st.info(
367
- """
368
- SolidPrivacy Scrub extends the demo with Dutch/EU and Dutch legal recognizers.
369
- For legal/confidential material, use fake documents in this public Space.
370
- The recognizer pack is designed to be local/offline compatible for a future desktop/MSI version.
371
- """
372
  )
373
 
374
- analyzer_load_state = st.info("Starting Presidio analyzer...")
375
- analyzer_load_state.empty()
376
-
377
  if st_recognition_profile == "Dutch Legal Strict":
378
  st.info(
379
- "Dutch Legal Strict mode is active. The app adds Dutch/EU recognizers plus legal/matter identifiers: "
380
- "zaaknummer, rolnummer, rekestnummer, parketnummer, dossiernummer, cliëntnummer, CJIB, ECLI, "
381
- "legal party references and court/authority references. It also shows possible unmasked "
382
- "reference candidates for review. Always review the editable replacement table."
383
  )
384
  elif st_recognition_profile == "Dutch / EU":
385
  st.info(
386
- "Dutch / EU mode is active. The app adds Dutch pattern recognizers for BSN, postcode, KvK, BTW/VAT, "
387
- "Dutch IBAN, Dutch phone numbers, addresses, license plates, rijbewijs-style numbers and BIG numbers. "
388
- "Always review the editable replacement table before exporting."
389
  )
390
 
391
- # Read default text
392
- with open("demo_text.txt", encoding="utf-8") as f:
393
- demo_text = f.readlines()
 
 
394
 
395
- st.subheader("Document input")
396
  uploaded_file = st.file_uploader(
397
- "Upload a .txt, .docx, or text-based .pdf file",
398
  type=["txt", "docx", "pdf"],
399
- help="For legal/confidential material, use only approved environments. This public demo should be used with fake or test documents.",
400
  )
401
 
402
  uploaded_file_type = None
403
  input_text = "".join(demo_text)
404
 
405
  if st_recognition_profile == "Dutch Legal Strict":
406
- with st.expander("Use a fake Dutch legal test example", expanded=False):
407
  example_names = get_example_names()
408
  if LEGAL_EXAMPLES_IMPORT_ERROR is not None:
409
  st.warning(
410
- "Could not import legal_test_examples.py. Showing embedded fallback examples. "
411
- "Check that legal_test_examples.py exists in the Space root. "
412
- f"Import error: {LEGAL_EXAMPLES_IMPORT_ERROR}"
413
  )
414
  elif not example_names:
415
- st.warning(
416
- "No legal examples were loaded from legal_test_examples.py. "
417
- "Check that TEST_CASES contains examples and that get_example_names() returns names."
418
- )
419
  example_names = list(EMBEDDED_LEGAL_TEST_CASES.keys())
420
 
421
  sample_name = st.selectbox(
422
- "Load synthetic legal example",
423
- ["Do not load a test example"] + example_names,
424
  index=0,
425
  )
426
- if sample_name != "Do not load a test example" and uploaded_file is None:
427
  example_text = get_example_text(sample_name)
428
  if not example_text and sample_name in EMBEDDED_LEGAL_TEST_CASES:
429
  example_text = EMBEDDED_LEGAL_TEST_CASES[sample_name]
430
  input_text = example_text
431
- st.caption("Loaded synthetic example text. No real personal data is included.")
432
 
433
  if uploaded_file is not None:
434
  try:
435
  input_text, uploaded_file_type = uploaded_file_to_text(uploaded_file)
436
- st.success(f"Loaded file: {uploaded_file.name}")
437
  except Exception as upload_error:
438
- st.error(f"Could not read uploaded file: {upload_error}")
439
 
440
  col1, col2 = st.columns(2)
441
- col1.subheader("Input")
442
  st_text = col1.text_area(
443
- label="Enter text or review extracted document text",
444
  value=input_text,
445
  height=400,
446
  key="text_input",
@@ -449,7 +433,6 @@ st_text = col1.text_area(
449
  try:
450
  all_supported_entities = list(get_supported_entities(*analyzer_params))
451
  general_dutch_entities = set(get_dutch_general_entity_names())
452
- legal_dutch_entities = set(get_dutch_legal_entity_names())
453
  all_dutch_entities = set(get_dutch_entity_names(include_legal=True))
454
 
455
  base_preferred_entities = {
@@ -474,26 +457,18 @@ try:
474
 
475
  default_entities = [entity for entity in all_supported_entities if entity in preferred_entities]
476
 
477
- st_entities_expander = st.sidebar.expander("Choose entities to look for")
478
- st_entities = st_entities_expander.multiselect(
479
- label="Which entities to look for?",
480
- options=all_supported_entities,
481
- default=default_entities,
482
- help=(
483
- "Dutch / EU mode adds recognizers such as NL_BSN, NL_POSTCODE, NL_KVK_NUMBER, "
484
- "NL_VAT_NUMBER, NL_IBAN and NL_PHONE_NUMBER. Dutch Legal Strict additionally adds "
485
- "NL_ECLI, NL_LEGAL_CASE_NUMBER, NL_PARKETNUMMER, NL_DOSSIER_NUMBER, NL_CLIENT_NUMBER, "
486
- "plus contextual references such as NL_CLIENT_REFERENCE, NL_SCHOOL_REFERENCE, "
487
- "NL_INVOICE_NUMBER, NL_CASE_REFERENCE and related legal/admin IDs."
488
- ),
489
- )
490
 
491
- analyzer_load_state = st.info("Starting Presidio analyzer...")
492
  analyzer = analyzer_engine(*analyzer_params)
493
  analyzer_load_state.empty()
494
 
495
- # The current demo uses English NER models. Dutch/EU pattern recognizers are
496
- # registered under language="en" so they can run without a separate Dutch NLP model.
497
  st_analyze_results = analyze(
498
  *analyzer_params,
499
  text=st_text,
@@ -507,7 +482,7 @@ try:
507
 
508
  if st_operator not in ("highlight", "synthesize"):
509
  with col2:
510
- st.subheader("Output")
511
  st_anonymize_results = anonymize(
512
  text=st_text,
513
  operator=st_operator,
@@ -516,7 +491,7 @@ try:
516
  encrypt_key=st_encrypt_key,
517
  analyze_results=st_analyze_results,
518
  )
519
- st.text_area(label="De-identified", value=st_anonymize_results.text, height=400)
520
 
521
  _, report_rows = build_placeholder_replacements(st_text, st_analyze_results)
522
  candidate_rows = []
@@ -524,36 +499,38 @@ try:
524
  candidate_rows = scan_unmasked_candidates(st_text, st_analyze_results, max_candidates=50)
525
 
526
  st.divider()
527
- st.subheader("Review replacement table before export")
528
  st.caption(
529
- "Untick false positives, change placeholders, add your own word pairs, "
530
- "and tick Remember for pairs you want to reuse in future documents. "
531
- "Candidate rows are suggestions and are unchecked by default."
532
  )
533
 
534
  if st_recognition_profile == "Dutch Legal Strict":
535
- with st.expander("Audit: possible unmasked candidates", expanded=bool(candidate_rows)):
536
  if candidate_rows:
537
  st.warning(
538
- "These values were not automatically masked, but look like possible legal/admin references. "
539
- "They are added to the review table below as unchecked suggestions."
540
  )
541
  candidate_display_df = pd.DataFrame(candidate_rows)
 
 
542
  candidate_display_df = candidate_display_df[
543
- ["entity_type", "text", "placeholder", "score", "reason", "context"]
544
  ].rename(
545
  columns={
546
- "entity_type": "Candidate type",
547
- "text": "Text",
548
- "placeholder": "Suggested replacement",
549
- "score": "Score",
550
- "reason": "Reason",
551
- "context": "Nearby context",
552
  }
553
  )
554
  st.dataframe(candidate_display_df, use_container_width=True)
555
  else:
556
- st.success("No suspicious unmasked reference candidates found by the audit layer.")
557
 
558
  remembered_rows = load_remembered_replacements()
559
  default_editor_rows = []
@@ -564,16 +541,20 @@ try:
564
  replace_with = str(row.get("replace_with", "")).strip()
565
  if not find_text or not replace_with:
566
  continue
 
567
  default_editor_rows.append(
568
  {
569
  "include": row.get("include", True),
570
  "remember": row.get("remember", True),
571
  "find": find_text,
572
  "replace_with": replace_with,
573
- "entity_type": row.get("entity_type", "REMEMBERED"),
 
 
574
  "score": None,
 
575
  "source": "remembered",
576
- "reason": "Saved reusable replacement",
577
  "context": "",
578
  }
579
  )
@@ -583,34 +564,45 @@ try:
583
  find_text = str(row.get("detected_text", "")).strip()
584
  if not find_text or find_text in seen_find_values:
585
  continue
 
 
586
  default_editor_rows.append(
587
  {
588
  "include": True,
589
  "remember": False,
590
  "find": find_text,
591
  "replace_with": row.get("placeholder", ""),
592
- "entity_type": row.get("entity_type", ""),
593
- "score": row.get("score", None),
 
 
 
594
  "source": "detected",
595
- "reason": "Automatically detected by recognizer",
596
  "context": "",
597
  }
598
  )
 
599
 
600
  for candidate in candidate_rows:
601
  find_text = str(candidate.get("text", "")).strip()
602
  if not find_text or find_text in seen_find_values:
603
  continue
 
 
604
  default_editor_rows.append(
605
  {
606
  "include": False,
607
  "remember": False,
608
  "find": find_text,
609
  "replace_with": candidate.get("placeholder", "<MOGELIJKE_REFERENTIE>"),
610
- "entity_type": candidate.get("entity_type", "NL_SUSPICIOUS_REFERENCE_CANDIDATE"),
611
- "score": candidate.get("score", None),
 
 
 
612
  "source": "candidate",
613
- "reason": candidate.get("reason", "Possible unmasked value"),
614
  "context": candidate.get("context", ""),
615
  }
616
  )
@@ -623,10 +615,13 @@ try:
623
  "remember": False,
624
  "find": "",
625
  "replace_with": "",
 
626
  "entity_type": "MANUAL",
 
627
  "score": None,
 
628
  "source": "manual",
629
- "reason": "Manual replacement row",
630
  "context": "",
631
  }
632
  ]
@@ -637,63 +632,59 @@ try:
637
  hide_index=True,
638
  num_rows="dynamic",
639
  use_container_width=True,
640
- column_order=["include", "remember", "find", "replace_with", "entity_type", "score", "source", "reason", "context"],
 
 
 
 
 
 
 
 
 
 
 
 
 
641
  column_config={
642
  "include": st.column_config.CheckboxColumn(
643
- "Use", help="Untick to exclude this replacement from the export.", default=True
644
  ),
645
  "remember": st.column_config.CheckboxColumn(
646
- "Remember", help="Save this replacement pair for future documents/sessions.", default=False
647
  ),
648
  "find": st.column_config.TextColumn(
649
- "Find text", help="The exact text that should be replaced."
650
  ),
651
  "replace_with": st.column_config.TextColumn(
652
- "Replace with", help="The placeholder to insert."
653
  ),
654
- "entity_type": st.column_config.TextColumn(
655
- "Entity type", help="Presidio entity type or MANUAL."
656
  ),
657
- "score": st.column_config.NumberColumn(
658
- "Score", help="Presidio/candidate confidence score, if available.", format="%.3f"
659
  ),
660
- "source": st.column_config.TextColumn(
661
- "Source", help="detected, candidate, remembered or manual."
662
  ),
663
  "reason": st.column_config.TextColumn(
664
- "Reason", help="Why this row was suggested or detected."
665
  ),
666
  "context": st.column_config.TextColumn(
667
- "Nearby context", help="Nearby text for candidate-review rows."
 
 
 
 
 
 
668
  ),
 
669
  },
670
  key="replacement_editor",
671
  )
672
 
673
- def safe_cell(value):
674
- if value is None:
675
- return ""
676
- try:
677
- if pd.isna(value):
678
- return ""
679
- except Exception:
680
- pass
681
- return str(value).strip()
682
-
683
- def safe_bool(value):
684
- if isinstance(value, bool):
685
- return value
686
- if value is None:
687
- return False
688
- try:
689
- if pd.isna(value):
690
- return False
691
- except Exception:
692
- pass
693
- if isinstance(value, (int, float)):
694
- return bool(value)
695
- return str(value).strip().lower() in ("true", "1", "yes", "y", "checked")
696
-
697
  edited_replacements = {}
698
  edited_report_rows = []
699
  for _, row in edited_replacements_df.iterrows():
@@ -716,13 +707,13 @@ try:
716
  }
717
  )
718
 
719
- st.info(f"{len(edited_replacements)} replacement pair(s) will be applied to the exports.")
720
 
721
  export_text = apply_replacements_to_text(st_text, edited_replacements)
722
- with st.expander("Preview anonymized text generated from edited table", expanded=False):
723
- st.text_area(label="Preview", value=export_text, height=300, key="edited_export_preview")
724
 
725
- st.subheader("Remember reusable replacements")
726
  remember_rows_to_save = []
727
  for _, row in edited_replacements_df.iterrows():
728
  include = safe_bool(row.get("include", False))
@@ -731,49 +722,47 @@ try:
731
  replace_text = safe_cell(row.get("replace_with", ""))
732
  entity_type = safe_cell(row.get("entity_type", "REMEMBERED")) or "REMEMBERED"
733
  if include and remember and find_text and replace_text:
734
- remember_rows_to_save.append(
735
- {"find": find_text, "replace_with": replace_text, "entity_type": entity_type}
736
- )
737
 
738
  memory_col1, memory_col2 = st.columns(2)
739
  with memory_col1:
740
- if st.button("Save remembered replacements"):
741
  saved_count = save_remembered_replacements(remember_rows_to_save)
742
- st.success(f"Saved {saved_count} remembered replacement pair(s).")
743
- st.info(f"Memory file: {get_memory_file_path()}")
744
  with memory_col2:
745
- if st.button("Clear remembered replacements"):
746
  clear_remembered_replacements()
747
- st.warning("Remembered replacements cleared.")
748
 
749
- st.subheader("Export anonymized files")
750
  if uploaded_file is not None:
751
- st.info(f"Uploaded file detected for export: {uploaded_file.name}")
752
  else:
753
- st.info("No uploaded file detected for export. Exporting from text area only.")
754
 
755
  st.download_button(
756
- label="Download anonymized text (.txt)",
757
  data=export_text.encode("utf-8"),
758
- file_name="anonymized_text.txt",
759
  mime="text/plain",
760
  key="download_txt",
761
  )
762
  st.download_button(
763
- label="Download replacement table (.csv)",
764
  data=replacement_report_csv(edited_report_rows),
765
- file_name="replacement_table.csv",
766
  mime="text/csv",
767
  key="download_csv",
768
  )
769
  st.download_button(
770
- label="Download scrub report (.txt)",
771
  data=scrub_report_txt(
772
  edited_report_rows,
773
- profile=st_recognition_profile,
774
  source_filename=uploaded_file.name if uploaded_file is not None else None,
775
  ),
776
- file_name="scrub_report.txt",
777
  mime="text/plain",
778
  key="download_scrub_report",
779
  )
@@ -781,63 +770,67 @@ try:
781
  try:
782
  if uploaded_file is not None and uploaded_file.name.lower().endswith(".docx"):
783
  docx_bytes = anonymized_docx_from_original(uploaded_file, edited_replacements)
784
- docx_filename = "anonymized_" + uploaded_file.name
785
  else:
786
  docx_bytes = docx_from_text(export_text)
787
- docx_filename = "anonymized_text.docx"
788
  st.download_button(
789
- label="Download anonymized Word file (.docx)",
790
  data=docx_bytes,
791
  file_name=docx_filename,
792
  mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
793
  key="download_docx",
794
  )
795
  except Exception as docx_error:
796
- st.error(f"Could not create DOCX export: {docx_error}")
797
 
798
  try:
799
  st.download_button(
800
- label="Download anonymized PDF (.pdf)",
801
  data=pdf_from_text(export_text),
802
- file_name="anonymized_text.pdf",
803
  mime="application/pdf",
804
  key="download_pdf",
805
  )
806
  except Exception as pdf_error:
807
- st.error(f"Could not create PDF export: {pdf_error}")
808
 
809
  elif st_operator == "synthesize":
810
  with col2:
811
- st.subheader("OpenAI Generated output")
812
  fake_data = create_fake_data(st_text, st_analyze_results, open_ai_params)
813
- st.text_area(label="Synthetic data", value=fake_data, height=400)
814
  else:
815
- st.subheader("Highlighted")
816
  annotated_tokens = annotate(text=st_text, analyze_results=st_analyze_results)
817
  annotated_text(*annotated_tokens)
818
 
819
- st.subheader("Findings" if not st_return_decision_process else "Findings with decision factors")
820
- if st_analyze_results:
821
- df = pd.DataFrame.from_records([r.to_dict() for r in st_analyze_results])
822
- df["text"] = [st_text[res.start : res.end] for res in st_analyze_results]
823
- df_subset = df[["entity_type", "text", "start", "end", "score"]].rename(
824
- {
825
- "entity_type": "Entity type",
826
- "text": "Text",
827
- "start": "Start",
828
- "end": "End",
829
- "score": "Confidence",
830
- },
831
- axis=1,
832
- )
833
- if st_return_decision_process:
834
- analysis_explanation_df = pd.DataFrame.from_records(
835
- [r.analysis_explanation.to_dict() for r in st_analyze_results]
836
  )
837
- df_subset = pd.concat([df_subset, analysis_explanation_df], axis=1)
838
- st.dataframe(df_subset.reset_index(drop=True), use_container_width=True)
839
- else:
840
- st.text("No findings")
 
 
 
 
841
 
842
  except Exception as e:
843
  print(e)
 
1
+ """Streamlit app for SolidPrivacy Scrub Legal.
2
+
3
+ v9 Dutch Legal UI Layer:
4
+ - presents Scrub as a Dutch legal document scrubber instead of a technical demo;
5
+ - keeps recognizer/engine internals under the hood;
6
+ - adds Dutch workflow labels, Dutch review table labels and Dutch download labels;
7
+ - preserves the existing detection, audit-candidate and export workflow.
 
 
 
 
 
 
8
  """
9
 
10
+ from __future__ import annotations
11
+
12
  import ast
13
  import logging
14
  import os
 
47
  clear_remembered_replacements,
48
  get_memory_file_path,
49
  )
50
+ from ui_texts_nl import (
51
+ APP_TITLE,
52
+ APP_SUBTITLE,
53
+ APP_INTRO,
54
+ LOCAL_PROCESSING_NOTE,
55
+ PROFILE_HELP,
56
+ PROFILE_DESCRIPTIONS,
57
+ OPERATOR_LABELS,
58
+ OPERATOR_HELP,
59
+ ADVANCED_SETTINGS_HELP,
60
+ )
61
+ from display_labels_nl import entity_label, source_label, confidence_label
62
 
63
  try:
64
  from candidate_scanner import scan_unmasked_candidates
 
72
  get_dutch_general_entity_names,
73
  get_dutch_legal_entity_names,
74
  )
75
+ except Exception:
76
  def get_dutch_entity_names(include_legal=True):
77
  return []
78
 
 
82
  def get_dutch_legal_entity_names():
83
  return []
84
 
85
+
86
  LEGAL_EXAMPLES_IMPORT_ERROR = None
87
 
88
  EMBEDDED_LEGAL_TEST_CASES = {
89
+ "Fallback - referenties en administratieve nummers": """Clientnummer: CL-FAM-55201.
90
  De schoolreferentie is HRZ-SAM-2026-04.
91
  In het verslag van Stichting Horizonzorg wordt dezelfde referentie HRZ-SAM-2026-04 genoemd.
92
  De factuur met nummer FACT-2026-4481 is onbetaald gebleven.
 
94
  De zaakreferentie is ZK-WOON-55091.
95
  Het artikel 7:669 BW mag niet worden gemaskeerd.
96
  De datum 15-12-2026 mag niet als referentie worden gezien.
97
+ Het bedrag EUR 1.250,00 mag niet als referentie worden gezien.
98
  """,
99
  "Fallback - familierecht contextbehoud": """Aan de Rechtbank Amsterdam
100
 
 
107
  """,
108
  }
109
 
110
+ PROFILE_OPTIONS = {
111
+ "Juridische controle — streng": "Dutch Legal Strict",
112
+ "Algemene Nederlandse controle": "Dutch / EU",
113
+ "Algemene internationale controle": "General / International",
114
+ }
115
+ INTERNAL_PROFILE_TO_LABEL = {value: key for key, value in PROFILE_OPTIONS.items()}
116
+ OPERATOR_LABEL_TO_VALUE = {label: value for value, label in OPERATOR_LABELS.items()}
117
 
 
 
118
 
119
+ def _load_legal_test_cases_from_file():
120
+ """Load legal examples as data instead of importing the module."""
 
 
 
121
  examples_path = Path(__file__).with_name("legal_test_examples.py")
122
  if not examples_path.exists():
123
  raise FileNotFoundError(f"{examples_path} does not exist")
 
133
  and node.target.id == "TEST_CASES"
134
  )
135
  if is_test_cases:
136
+ cases = ast.literal_eval(node.value)
 
137
  if not isinstance(cases, list):
138
  raise ValueError("TEST_CASES is not a list")
139
  return cases
 
149
 
150
  def get_example_names():
151
  if TEST_CASES:
152
+ return [str(case.get("name", "Naamloos voorbeeld")) for case in TEST_CASES]
153
  return list(EMBEDDED_LEGAL_TEST_CASES.keys())
154
 
155
 
 
160
  return EMBEDDED_LEGAL_TEST_CASES.get(name, "")
161
 
162
 
163
+ def safe_cell(value):
164
+ if value is None:
165
+ return ""
166
+ try:
167
+ if pd.isna(value):
168
+ return ""
169
+ except Exception:
170
+ pass
171
+ return str(value).strip()
172
+
173
+
174
+ def safe_bool(value):
175
+ if isinstance(value, bool):
176
+ return value
177
+ if value is None:
178
+ return False
179
+ try:
180
+ if pd.isna(value):
181
+ return False
182
+ except Exception:
183
+ pass
184
+ if isinstance(value, (int, float)):
185
+ return bool(value)
186
+ return str(value).strip().lower() in ("true", "1", "yes", "y", "checked", "ja")
187
+
188
+
189
  st.set_page_config(
190
+ page_title=APP_TITLE,
191
  layout="wide",
192
  initial_sidebar_state="expanded",
193
+ menu_items={"About": "SolidPrivacy Scrub Legal"},
194
  )
195
 
196
  dotenv.load_dotenv()
197
+ logger = logging.getLogger("solidprivacy-scrub")
198
  allow_other_models = os.getenv("ALLOW_OTHER_MODELS", False)
199
 
200
+ st.sidebar.header(APP_TITLE)
201
+ st.sidebar.caption(APP_SUBTITLE)
202
 
203
+ profile_label = st.sidebar.selectbox(
204
+ "Controlemodus",
205
+ list(PROFILE_OPTIONS.keys()),
206
+ index=0,
207
+ help=PROFILE_HELP,
 
 
208
  )
209
+ st_recognition_profile = PROFILE_OPTIONS[profile_label]
210
+ st.sidebar.info(PROFILE_DESCRIPTIONS.get(profile_label, ""))
211
+
212
+ operator_label = st.sidebar.selectbox(
213
+ "Manier van vervangen",
214
+ list(OPERATOR_LABELS.values()),
215
+ index=list(OPERATOR_LABELS.keys()).index("replace"),
216
+ help=OPERATOR_HELP,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
217
  )
218
+ st_operator = OPERATOR_LABEL_TO_VALUE[operator_label]
219
 
220
+ st_threshold_default = 0.30 if st_recognition_profile == "Dutch Legal Strict" else 0.35
 
 
 
 
 
221
 
222
+ with st.sidebar.expander("Geavanceerde instellingen", expanded=False):
223
+ st.caption(ADVANCED_SETTINGS_HELP)
224
+ model_help_text = (
225
+ "Kies het NER-model dat naast regelherkenning wordt gebruikt. "
226
+ "De Nederlandse juridische herkenners zijn regelgebaseerd."
227
  )
228
+ st_ta_key = st_ta_endpoint = ""
229
+ model_list = [
230
+ "spaCy/en_core_web_lg",
231
+ "flair/ner-english-large",
232
+ "HuggingFace/obi/deid_roberta_i2b2",
233
+ "HuggingFace/StanfordAIMI/stanford-deidentifier-base",
234
+ "stanza/en",
235
+ "Azure AI Language",
236
+ "Other",
237
+ ]
238
+ if not allow_other_models:
239
+ model_list.pop()
240
+
241
+ st_model = st.selectbox(
242
+ "Technisch NER-model",
243
+ model_list,
244
+ index=1,
245
+ help=model_help_text,
246
  )
247
+
248
+ st_model_package = st_model.split("/")[0]
249
+ st_model = (
250
+ st_model
251
+ if st_model_package.lower() not in ("spacy", "stanza", "huggingface")
252
+ else "/".join(st_model.split("/")[1:])
253
  )
254
 
255
+ if st_model == "Other":
256
+ st_model_package = st.selectbox(
257
+ "NER-modelpakket", options=["spaCy", "stanza", "Flair", "HuggingFace"]
258
+ )
259
+ st_model = st.text_input("NER-modelnaam", value="")
260
 
261
+ if st_model == "Azure AI Language":
262
+ st_ta_key = st.text_input(
263
+ "Azure AI Language key", value=os.getenv("TA_KEY", ""), type="password"
264
+ )
265
+ st_ta_endpoint = st.text_input(
266
+ "Azure AI Language endpoint",
267
+ value=os.getenv("TA_ENDPOINT", default=""),
268
+ )
 
 
 
269
 
270
+ st_threshold = st.slider(
271
+ label="Gevoeligheid van herkenning",
272
+ min_value=0.0,
273
+ max_value=1.0,
274
+ value=st_threshold_default,
275
+ help="Lagere waarde = meer gevonden gegevens, maar ook meer kans op fout-positieven.",
276
+ )
277
+ st_return_decision_process = st.checkbox(
278
+ "Toon technische beslisinformatie",
279
+ value=False,
280
+ help="Voegt technische uitlegvelden toe aan de resultatentabel.",
281
+ )
282
+ st_mask_char = st.text_input("Maskeringsteken", value="*", max_chars=1)
283
+ st_number_of_chars = st.number_input("Aantal te maskeren tekens", value=15, min_value=0, max_value=100)
284
+ st_encrypt_key = st.text_input("AES-sleutel", value="WmZq4t7w!z%C&F)J")
285
+
286
+ st_deny_allow_expander = st.expander("Woordenlijsten", expanded=False)
287
+ with st_deny_allow_expander:
288
+ st_allow_list = st_tags(label="Niet vervangen", text="Voer woord in en druk op Enter.")
289
+ st.caption("Woorden in deze lijst worden niet als gevoelig gegeven behandeld.")
290
+ st_deny_list = st_tags(label="Extra controleren", text="Voer woord in en druk op Enter.")
291
+ st.caption("Woorden in deze lijst krijgen extra aandacht bij de herkenning.")
292
 
293
+ analyzer_params = (st_model_package, st_model, st_ta_key, st_ta_endpoint)
 
 
294
  open_ai_params = None
 
295
 
296
 
297
  def set_up_openai_synthesis():
 
298
  if os.getenv("OPENAI_TYPE", default="openai") == "Azure":
299
  openai_api_type = "azure"
300
  st_openai_api_base = st.sidebar.text_input(
 
313
  st_deployment_id = ""
314
  openai_key = os.getenv("OPENAI_KEY", default="")
315
 
316
+ st_openai_key = st.sidebar.text_input("OPENAI_KEY", value=openai_key, type="password")
 
 
 
 
 
317
  st_openai_model = st.sidebar.text_input(
318
+ "OpenAI-model voor synthetische tekst",
319
  value=os.getenv("OPENAI_MODEL", default="gpt-3.5-turbo-instruct"),
 
320
  )
321
  return (
322
  openai_api_type,
 
328
  )
329
 
330
 
331
+ if st_operator == "synthesize":
 
 
 
 
 
 
 
332
  (
333
  openai_api_type,
334
  st_openai_api_base,
 
346
  api_type=openai_api_type,
347
  )
348
 
349
+ st.title(APP_TITLE)
350
+ st.subheader(APP_SUBTITLE)
351
+ st.write(APP_INTRO)
352
+ st.info(LOCAL_PROCESSING_NOTE)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
353
 
354
+ with st.expander("Over deze app", expanded=False):
355
+ st.write(
356
+ "Scrub Legal helpt bij het controleerbaar opschonen van juridische tekst. "
357
+ "De herkenning combineert algemene patroonherkenning, Nederlandse herkenners, "
358
+ "juridische referentietaxonomie en een auditlaag voor mogelijke gemiste waarden."
 
 
 
 
 
 
359
  )
360
+ st.write(
361
+ "De technische detectie-engine blijft onder de motorkap. De gebruiker beoordeelt "
362
+ "altijd zelf de gevonden gegevens en mogelijke kandidaten in de vervangtabel."
 
 
 
363
  )
364
 
 
 
 
365
  if st_recognition_profile == "Dutch Legal Strict":
366
  st.info(
367
+ "Juridische controle is actief. Scrub zoekt extra naar zaaknummers, rolnummers, "
368
+ "rekestnummers, parketnummers, dossiernummers, clientnummers, CJIB, ECLI, "
369
+ "procespartijen, instanties en mogelijke juridische referenties."
 
370
  )
371
  elif st_recognition_profile == "Dutch / EU":
372
  st.info(
373
+ "Algemene Nederlandse controle is actief. Scrub zoekt onder meer naar BSN, postcode, "
374
+ "KvK, btw-nummer, Nederlandse IBAN, telefoonnummers, adressen, kentekens en BIG-nummers."
 
375
  )
376
 
377
+ try:
378
+ with open("demo_text.txt", encoding="utf-8") as f:
379
+ demo_text = f.readlines()
380
+ except Exception:
381
+ demo_text = ["Plak of upload hier tekst om te controleren."]
382
 
383
+ st.subheader("1. Voeg document of tekst toe")
384
  uploaded_file = st.file_uploader(
385
+ "Upload een .txt-, .docx- of tekstgebaseerd .pdf-bestand",
386
  type=["txt", "docx", "pdf"],
387
+ help="Gebruik in deze publieke prototypeomgeving alleen synthetische of goedgekeurde testdocumenten.",
388
  )
389
 
390
  uploaded_file_type = None
391
  input_text = "".join(demo_text)
392
 
393
  if st_recognition_profile == "Dutch Legal Strict":
394
+ with st.expander("Gebruik een synthetisch juridisch testvoorbeeld", expanded=False):
395
  example_names = get_example_names()
396
  if LEGAL_EXAMPLES_IMPORT_ERROR is not None:
397
  st.warning(
398
+ "Kon legal_test_examples.py niet laden. Ingebouwde fallback-voorbeelden worden getoond. "
399
+ f"Foutmelding: {LEGAL_EXAMPLES_IMPORT_ERROR}"
 
400
  )
401
  elif not example_names:
402
+ st.warning("Er zijn geen juridische voorbeelden geladen.")
 
 
 
403
  example_names = list(EMBEDDED_LEGAL_TEST_CASES.keys())
404
 
405
  sample_name = st.selectbox(
406
+ "Laad synthetisch juridisch voorbeeld",
407
+ ["Geen testvoorbeeld laden"] + example_names,
408
  index=0,
409
  )
410
+ if sample_name != "Geen testvoorbeeld laden" and uploaded_file is None:
411
  example_text = get_example_text(sample_name)
412
  if not example_text and sample_name in EMBEDDED_LEGAL_TEST_CASES:
413
  example_text = EMBEDDED_LEGAL_TEST_CASES[sample_name]
414
  input_text = example_text
415
+ st.caption("Synthetische voorbeeldtekst geladen. Er staan geen echte persoonsgegevens in.")
416
 
417
  if uploaded_file is not None:
418
  try:
419
  input_text, uploaded_file_type = uploaded_file_to_text(uploaded_file)
420
+ st.success(f"Bestand geladen: {uploaded_file.name}")
421
  except Exception as upload_error:
422
+ st.error(f"Kon het bestand niet lezen: {upload_error}")
423
 
424
  col1, col2 = st.columns(2)
425
+ col1.subheader("Invoer")
426
  st_text = col1.text_area(
427
+ label="Plak tekst of controleer de uit het document gehaalde tekst",
428
  value=input_text,
429
  height=400,
430
  key="text_input",
 
433
  try:
434
  all_supported_entities = list(get_supported_entities(*analyzer_params))
435
  general_dutch_entities = set(get_dutch_general_entity_names())
 
436
  all_dutch_entities = set(get_dutch_entity_names(include_legal=True))
437
 
438
  base_preferred_entities = {
 
457
 
458
  default_entities = [entity for entity in all_supported_entities if entity in preferred_entities]
459
 
460
+ with st.sidebar.expander("Te herkennen gegevenstypen", expanded=False):
461
+ st_entities = st.multiselect(
462
+ label="Welke typen gegevens moet Scrub zoeken?",
463
+ options=all_supported_entities,
464
+ default=default_entities,
465
+ help="Laat dit standaard staan, tenzij je gericht wilt testen of tunen.",
466
+ )
 
 
 
 
 
 
467
 
468
+ analyzer_load_state = st.info("Herkenningsengine starten...")
469
  analyzer = analyzer_engine(*analyzer_params)
470
  analyzer_load_state.empty()
471
 
 
 
472
  st_analyze_results = analyze(
473
  *analyzer_params,
474
  text=st_text,
 
482
 
483
  if st_operator not in ("highlight", "synthesize"):
484
  with col2:
485
+ st.subheader("Directe voorbeeldweergave")
486
  st_anonymize_results = anonymize(
487
  text=st_text,
488
  operator=st_operator,
 
491
  encrypt_key=st_encrypt_key,
492
  analyze_results=st_analyze_results,
493
  )
494
+ st.text_area(label="Automatisch resultaat", value=st_anonymize_results.text, height=400)
495
 
496
  _, report_rows = build_placeholder_replacements(st_text, st_analyze_results)
497
  candidate_rows = []
 
499
  candidate_rows = scan_unmasked_candidates(st_text, st_analyze_results, max_candidates=50)
500
 
501
  st.divider()
502
+ st.subheader("2. Controleer gevonden gegevens")
503
  st.caption(
504
+ "Vink fout-positieven uit, pas placeholders aan, voeg handmatige vervangingen toe "
505
+ "en vink Onthouden aan voor vervangingen die je opnieuw wilt gebruiken. "
506
+ "Mogelijke kandidaten staan standaard uitgevinkt."
507
  )
508
 
509
  if st_recognition_profile == "Dutch Legal Strict":
510
+ with st.expander("Mogelijke gemiste waarden", expanded=bool(candidate_rows)):
511
  if candidate_rows:
512
  st.warning(
513
+ "Deze waarden zijn niet automatisch vervangen, maar lijken mogelijk op juridische of administratieve referenties. "
514
+ "Controleer ze en vink ze alleen aan als ze echt vervangen moeten worden."
515
  )
516
  candidate_display_df = pd.DataFrame(candidate_rows)
517
+ candidate_display_df["type_gegeven"] = candidate_display_df["entity_type"].map(entity_label)
518
+ candidate_display_df["zekerheid"] = candidate_display_df["score"].map(confidence_label)
519
  candidate_display_df = candidate_display_df[
520
+ ["type_gegeven", "text", "placeholder", "zekerheid", "reason", "context"]
521
  ].rename(
522
  columns={
523
+ "type_gegeven": "Type gegeven",
524
+ "text": "Gevonden tekst",
525
+ "placeholder": "Voorgestelde vervanging",
526
+ "zekerheid": "Zekerheid",
527
+ "reason": "Reden",
528
+ "context": "Context",
529
  }
530
  )
531
  st.dataframe(candidate_display_df, use_container_width=True)
532
  else:
533
+ st.success("Geen mogelijke gemiste referenties gevonden door de auditlaag.")
534
 
535
  remembered_rows = load_remembered_replacements()
536
  default_editor_rows = []
 
541
  replace_with = str(row.get("replace_with", "")).strip()
542
  if not find_text or not replace_with:
543
  continue
544
+ entity_type = row.get("entity_type", "REMEMBERED")
545
  default_editor_rows.append(
546
  {
547
  "include": row.get("include", True),
548
  "remember": row.get("remember", True),
549
  "find": find_text,
550
  "replace_with": replace_with,
551
+ "type_label": entity_label(entity_type),
552
+ "entity_type": entity_type,
553
+ "confidence": "",
554
  "score": None,
555
+ "source_label": source_label("remembered"),
556
  "source": "remembered",
557
+ "reason": "Opgeslagen herbruikbare vervanging",
558
  "context": "",
559
  }
560
  )
 
564
  find_text = str(row.get("detected_text", "")).strip()
565
  if not find_text or find_text in seen_find_values:
566
  continue
567
+ entity_type = row.get("entity_type", "")
568
+ score = row.get("score", None)
569
  default_editor_rows.append(
570
  {
571
  "include": True,
572
  "remember": False,
573
  "find": find_text,
574
  "replace_with": row.get("placeholder", ""),
575
+ "type_label": entity_label(entity_type),
576
+ "entity_type": entity_type,
577
+ "confidence": confidence_label(score),
578
+ "score": score,
579
+ "source_label": source_label("detected"),
580
  "source": "detected",
581
+ "reason": "Automatisch herkend",
582
  "context": "",
583
  }
584
  )
585
+ seen_find_values.add(find_text)
586
 
587
  for candidate in candidate_rows:
588
  find_text = str(candidate.get("text", "")).strip()
589
  if not find_text or find_text in seen_find_values:
590
  continue
591
+ entity_type = candidate.get("entity_type", "NL_SUSPICIOUS_REFERENCE_CANDIDATE")
592
+ score = candidate.get("score", None)
593
  default_editor_rows.append(
594
  {
595
  "include": False,
596
  "remember": False,
597
  "find": find_text,
598
  "replace_with": candidate.get("placeholder", "<MOGELIJKE_REFERENTIE>"),
599
+ "type_label": entity_label(entity_type),
600
+ "entity_type": entity_type,
601
+ "confidence": confidence_label(score),
602
+ "score": score,
603
+ "source_label": source_label("candidate"),
604
  "source": "candidate",
605
+ "reason": candidate.get("reason", "Mogelijke gemiste waarde"),
606
  "context": candidate.get("context", ""),
607
  }
608
  )
 
615
  "remember": False,
616
  "find": "",
617
  "replace_with": "",
618
+ "type_label": entity_label("MANUAL"),
619
  "entity_type": "MANUAL",
620
+ "confidence": "",
621
  "score": None,
622
+ "source_label": source_label("manual"),
623
  "source": "manual",
624
+ "reason": "Handmatige vervangingsregel",
625
  "context": "",
626
  }
627
  ]
 
632
  hide_index=True,
633
  num_rows="dynamic",
634
  use_container_width=True,
635
+ column_order=[
636
+ "include",
637
+ "remember",
638
+ "find",
639
+ "replace_with",
640
+ "type_label",
641
+ "confidence",
642
+ "source_label",
643
+ "reason",
644
+ "context",
645
+ "entity_type",
646
+ "score",
647
+ "source",
648
+ ],
649
  column_config={
650
  "include": st.column_config.CheckboxColumn(
651
+ "Meenemen", help="Vink uit om deze vervanging niet toe te passen.", default=True
652
  ),
653
  "remember": st.column_config.CheckboxColumn(
654
+ "Onthouden", help="Bewaar deze vervanging voor later gebruik.", default=False
655
  ),
656
  "find": st.column_config.TextColumn(
657
+ "Gevonden tekst", help="Exacte tekst die vervangen moet worden."
658
  ),
659
  "replace_with": st.column_config.TextColumn(
660
+ "Vervangen door", help="Placeholder of vervangende tekst."
661
  ),
662
+ "type_label": st.column_config.TextColumn(
663
+ "Type gegeven", help="Gebruiksvriendelijke categorie."
664
  ),
665
+ "confidence": st.column_config.TextColumn(
666
+ "Zekerheid", help="Globale inschatting van de herkenningszekerheid."
667
  ),
668
+ "source_label": st.column_config.TextColumn(
669
+ "Bron", help="Automatisch herkend, mogelijke kandidaat, onthouden of handmatig."
670
  ),
671
  "reason": st.column_config.TextColumn(
672
+ "Reden", help="Waarom deze regel is voorgesteld."
673
  ),
674
  "context": st.column_config.TextColumn(
675
+ "Context", help="Nabije tekst voor kandidaatregels."
676
+ ),
677
+ "entity_type": st.column_config.TextColumn(
678
+ "Technisch type", help="Interne herkennercategorie."
679
+ ),
680
+ "score": st.column_config.NumberColumn(
681
+ "Technische score", help="Numerieke score, indien beschikbaar.", format="%.3f"
682
  ),
683
+ "source": st.column_config.TextColumn("Technische bron"),
684
  },
685
  key="replacement_editor",
686
  )
687
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
688
  edited_replacements = {}
689
  edited_report_rows = []
690
  for _, row in edited_replacements_df.iterrows():
 
707
  }
708
  )
709
 
710
+ st.info(f"{len(edited_replacements)} vervanging(en) worden toegepast op de exports.")
711
 
712
  export_text = apply_replacements_to_text(st_text, edited_replacements)
713
+ with st.expander("Voorbeeld op basis van de gecontroleerde vervangtabel", expanded=False):
714
+ st.text_area(label="Gecontroleerde voorbeeldtekst", value=export_text, height=300, key="edited_export_preview")
715
 
716
+ st.subheader("3. Onthoud herbruikbare vervangingen")
717
  remember_rows_to_save = []
718
  for _, row in edited_replacements_df.iterrows():
719
  include = safe_bool(row.get("include", False))
 
722
  replace_text = safe_cell(row.get("replace_with", ""))
723
  entity_type = safe_cell(row.get("entity_type", "REMEMBERED")) or "REMEMBERED"
724
  if include and remember and find_text and replace_text:
725
+ remember_rows_to_save.append({"find": find_text, "replace_with": replace_text, "entity_type": entity_type})
 
 
726
 
727
  memory_col1, memory_col2 = st.columns(2)
728
  with memory_col1:
729
+ if st.button("Onthouden vervangingen opslaan"):
730
  saved_count = save_remembered_replacements(remember_rows_to_save)
731
+ st.success(f"{saved_count} vervanging(en) opgeslagen.")
732
+ st.info(f"Geheugenbestand: {get_memory_file_path()}")
733
  with memory_col2:
734
+ if st.button("Onthouden vervangingen wissen"):
735
  clear_remembered_replacements()
736
+ st.warning("Onthouden vervangingen gewist.")
737
 
738
+ st.subheader("4. Download opgeschoonde bestanden")
739
  if uploaded_file is not None:
740
+ st.info(f"Bestand beschikbaar voor export: {uploaded_file.name}")
741
  else:
742
+ st.info("Geen uploadbestand aanwezig. Export wordt gemaakt op basis van het tekstvak.")
743
 
744
  st.download_button(
745
+ label="Download opgeschoonde tekst (.txt)",
746
  data=export_text.encode("utf-8"),
747
+ file_name="opgeschoonde_tekst.txt",
748
  mime="text/plain",
749
  key="download_txt",
750
  )
751
  st.download_button(
752
+ label="Download vervangtabel (.csv)",
753
  data=replacement_report_csv(edited_report_rows),
754
+ file_name="vervangtabel.csv",
755
  mime="text/csv",
756
  key="download_csv",
757
  )
758
  st.download_button(
759
+ label="Download scrubrapport (.txt)",
760
  data=scrub_report_txt(
761
  edited_report_rows,
762
+ profile=profile_label,
763
  source_filename=uploaded_file.name if uploaded_file is not None else None,
764
  ),
765
+ file_name="scrubrapport.txt",
766
  mime="text/plain",
767
  key="download_scrub_report",
768
  )
 
770
  try:
771
  if uploaded_file is not None and uploaded_file.name.lower().endswith(".docx"):
772
  docx_bytes = anonymized_docx_from_original(uploaded_file, edited_replacements)
773
+ docx_filename = "opgeschoond_" + uploaded_file.name
774
  else:
775
  docx_bytes = docx_from_text(export_text)
776
+ docx_filename = "opgeschoonde_tekst.docx"
777
  st.download_button(
778
+ label="Download opgeschoond Word-bestand (.docx)",
779
  data=docx_bytes,
780
  file_name=docx_filename,
781
  mime="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
782
  key="download_docx",
783
  )
784
  except Exception as docx_error:
785
+ st.error(f"Kon geen DOCX-export maken: {docx_error}")
786
 
787
  try:
788
  st.download_button(
789
+ label="Download opgeschoonde PDF (.pdf)",
790
  data=pdf_from_text(export_text),
791
+ file_name="opgeschoonde_tekst.pdf",
792
  mime="application/pdf",
793
  key="download_pdf",
794
  )
795
  except Exception as pdf_error:
796
+ st.error(f"Kon geen PDF-export maken: {pdf_error}")
797
 
798
  elif st_operator == "synthesize":
799
  with col2:
800
+ st.subheader("Synthetische tekst")
801
  fake_data = create_fake_data(st_text, st_analyze_results, open_ai_params)
802
+ st.text_area(label="Synthetische data", value=fake_data, height=400)
803
  else:
804
+ st.subheader("Gemarkeerde tekst")
805
  annotated_tokens = annotate(text=st_text, analyze_results=st_analyze_results)
806
  annotated_text(*annotated_tokens)
807
 
808
+ with st.expander("Technische herkenningen", expanded=False):
809
+ if st_analyze_results:
810
+ df = pd.DataFrame.from_records([r.to_dict() for r in st_analyze_results])
811
+ df["text"] = [st_text[res.start : res.end] for res in st_analyze_results]
812
+ df["type_gegeven"] = df["entity_type"].map(entity_label)
813
+ df["zekerheid"] = df["score"].map(confidence_label)
814
+ df_subset = df[["type_gegeven", "text", "start", "end", "score", "zekerheid", "entity_type"]].rename(
815
+ {
816
+ "type_gegeven": "Type gegeven",
817
+ "text": "Gevonden tekst",
818
+ "start": "Start",
819
+ "end": "Einde",
820
+ "score": "Score",
821
+ "zekerheid": "Zekerheid",
822
+ "entity_type": "Technisch type",
823
+ },
824
+ axis=1,
825
  )
826
+ if st_return_decision_process:
827
+ analysis_explanation_df = pd.DataFrame.from_records(
828
+ [r.analysis_explanation.to_dict() for r in st_analyze_results]
829
+ )
830
+ df_subset = pd.concat([df_subset, analysis_explanation_df], axis=1)
831
+ st.dataframe(df_subset.reset_index(drop=True), use_container_width=True)
832
+ else:
833
+ st.text("Geen herkenningen gevonden.")
834
 
835
  except Exception as e:
836
  print(e)