solidprivacy commited on
Commit
cba709c
·
verified ·
1 Parent(s): 0256326

Add Dutch EU profile to Streamlit app

Browse files
Files changed (1) hide show
  1. presidio_streamlit.py +52 -4
presidio_streamlit.py CHANGED
@@ -37,6 +37,12 @@ from replacement_memory import (
37
  get_memory_file_path,
38
  )
39
 
 
 
 
 
 
 
40
  st.set_page_config(
41
  page_title="Presidio demo",
42
  layout="wide",
@@ -119,6 +125,16 @@ st.sidebar.warning("Note: Models might take some time to download. ")
119
  analyzer_params = (st_model_package, st_model, st_ta_key, st_ta_endpoint)
120
  logger.debug(f"analyzer_params: {analyzer_params}")
121
 
 
 
 
 
 
 
 
 
 
 
122
  st_operator = st.sidebar.selectbox(
123
  "De-identification approach",
124
  ["redact", "replace", "synthesize", "highlight", "mask", "hash", "encrypt"],
@@ -285,6 +301,14 @@ analyzer_load_state = st.info("Starting Presidio analyzer...")
285
 
286
  analyzer_load_state.empty()
287
 
 
 
 
 
 
 
 
 
288
  # Read default text
289
  with open("demo_text.txt") as f:
290
  demo_text = f.readlines()
@@ -322,14 +346,35 @@ st_text = col1.text_area(
322
 
323
  try:
324
  # Choose entities
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
325
  st_entities_expander = st.sidebar.expander("Choose entities to look for")
326
  st_entities = st_entities_expander.multiselect(
327
  label="Which entities to look for?",
328
- options=get_supported_entities(*analyzer_params),
329
- default=list(get_supported_entities(*analyzer_params)),
330
  help="Limit the list of PII entities detected. "
331
- "This list is dynamic and based on the NER model and registered recognizers. "
332
- "More information can be found here: https://microsoft.github.io/presidio/analyzer/adding_recognizers/",
333
  )
334
 
335
  # Before
@@ -337,6 +382,9 @@ try:
337
  analyzer = analyzer_engine(*analyzer_params)
338
  analyzer_load_state.empty()
339
 
 
 
 
340
  st_analyze_results = analyze(
341
  *analyzer_params,
342
  text=st_text,
 
37
  get_memory_file_path,
38
  )
39
 
40
+ try:
41
+ from dutch_recognizers import get_dutch_entity_names
42
+ except Exception: # keep app usable while the new file is being added
43
+ def get_dutch_entity_names():
44
+ return []
45
+
46
  st.set_page_config(
47
  page_title="Presidio demo",
48
  layout="wide",
 
125
  analyzer_params = (st_model_package, st_model, st_ta_key, st_ta_endpoint)
126
  logger.debug(f"analyzer_params: {analyzer_params}")
127
 
128
+ st_recognition_profile = st.sidebar.selectbox(
129
+ "Recognition profile",
130
+ ["Dutch / EU", "General / International"],
131
+ index=0,
132
+ help=(
133
+ "Dutch / EU enables Dutch pattern recognizers such as BSN, postcode, "
134
+ "KvK, BTW/VAT, Dutch IBAN, Dutch phone numbers and Dutch license plates."
135
+ ),
136
+ )
137
+
138
  st_operator = st.sidebar.selectbox(
139
  "De-identification approach",
140
  ["redact", "replace", "synthesize", "highlight", "mask", "hash", "encrypt"],
 
301
 
302
  analyzer_load_state.empty()
303
 
304
+ if st_recognition_profile == "Dutch / EU":
305
+ st.info(
306
+ "Dutch / EU mode is active. The app adds Dutch pattern recognizers "
307
+ "for BSN, postcode, KvK, BTW/VAT, Dutch IBAN, Dutch phone numbers, "
308
+ "license plates, rijbewijs-style numbers and BIG numbers. Always review "
309
+ "the editable replacement table before exporting."
310
+ )
311
+
312
  # Read default text
313
  with open("demo_text.txt") as f:
314
  demo_text = f.readlines()
 
346
 
347
  try:
348
  # Choose entities
349
+ all_supported_entities = list(get_supported_entities(*analyzer_params))
350
+ dutch_entities = set(get_dutch_entity_names())
351
+
352
+ if st_recognition_profile == "Dutch / EU":
353
+ preferred_entities = {
354
+ "PERSON",
355
+ "LOCATION",
356
+ "ORGANIZATION",
357
+ "EMAIL_ADDRESS",
358
+ "PHONE_NUMBER",
359
+ "IBAN_CODE",
360
+ "URL",
361
+ "IP_ADDRESS",
362
+ "GENERIC_PII",
363
+ } | dutch_entities
364
+ default_entities = [
365
+ entity for entity in all_supported_entities if entity in preferred_entities
366
+ ]
367
+ else:
368
+ default_entities = list(all_supported_entities)
369
+
370
  st_entities_expander = st.sidebar.expander("Choose entities to look for")
371
  st_entities = st_entities_expander.multiselect(
372
  label="Which entities to look for?",
373
+ options=all_supported_entities,
374
+ default=default_entities,
375
  help="Limit the list of PII entities detected. "
376
+ "Dutch / EU mode adds recognizers such as NL_BSN, NL_POSTCODE, "
377
+ "NL_KVK_NUMBER, NL_VAT_NUMBER, NL_IBAN and NL_PHONE_NUMBER.",
378
  )
379
 
380
  # Before
 
382
  analyzer = analyzer_engine(*analyzer_params)
383
  analyzer_load_state.empty()
384
 
385
+ # The current demo uses English NER models. Dutch/EU pattern recognizers
386
+ # are registered under language="en" so they can run without requiring a
387
+ # separate Dutch NLP model.
388
  st_analyze_results = analyze(
389
  *analyzer_params,
390
  text=st_text,