Spaces:
Running
Running
Add Dutch EU profile to Streamlit app
Browse files- presidio_streamlit.py +52 -4
presidio_streamlit.py
CHANGED
|
@@ -37,6 +37,12 @@ from replacement_memory import (
|
|
| 37 |
get_memory_file_path,
|
| 38 |
)
|
| 39 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 40 |
st.set_page_config(
|
| 41 |
page_title="Presidio demo",
|
| 42 |
layout="wide",
|
|
@@ -119,6 +125,16 @@ st.sidebar.warning("Note: Models might take some time to download. ")
|
|
| 119 |
analyzer_params = (st_model_package, st_model, st_ta_key, st_ta_endpoint)
|
| 120 |
logger.debug(f"analyzer_params: {analyzer_params}")
|
| 121 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
st_operator = st.sidebar.selectbox(
|
| 123 |
"De-identification approach",
|
| 124 |
["redact", "replace", "synthesize", "highlight", "mask", "hash", "encrypt"],
|
|
@@ -285,6 +301,14 @@ analyzer_load_state = st.info("Starting Presidio analyzer...")
|
|
| 285 |
|
| 286 |
analyzer_load_state.empty()
|
| 287 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
# Read default text
|
| 289 |
with open("demo_text.txt") as f:
|
| 290 |
demo_text = f.readlines()
|
|
@@ -322,14 +346,35 @@ st_text = col1.text_area(
|
|
| 322 |
|
| 323 |
try:
|
| 324 |
# Choose entities
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 325 |
st_entities_expander = st.sidebar.expander("Choose entities to look for")
|
| 326 |
st_entities = st_entities_expander.multiselect(
|
| 327 |
label="Which entities to look for?",
|
| 328 |
-
options=
|
| 329 |
-
default=
|
| 330 |
help="Limit the list of PII entities detected. "
|
| 331 |
-
"
|
| 332 |
-
"
|
| 333 |
)
|
| 334 |
|
| 335 |
# Before
|
|
@@ -337,6 +382,9 @@ try:
|
|
| 337 |
analyzer = analyzer_engine(*analyzer_params)
|
| 338 |
analyzer_load_state.empty()
|
| 339 |
|
|
|
|
|
|
|
|
|
|
| 340 |
st_analyze_results = analyze(
|
| 341 |
*analyzer_params,
|
| 342 |
text=st_text,
|
|
|
|
| 37 |
get_memory_file_path,
|
| 38 |
)
|
| 39 |
|
| 40 |
+
try:
|
| 41 |
+
from dutch_recognizers import get_dutch_entity_names
|
| 42 |
+
except Exception: # keep app usable while the new file is being added
|
| 43 |
+
def get_dutch_entity_names():
|
| 44 |
+
return []
|
| 45 |
+
|
| 46 |
st.set_page_config(
|
| 47 |
page_title="Presidio demo",
|
| 48 |
layout="wide",
|
|
|
|
| 125 |
analyzer_params = (st_model_package, st_model, st_ta_key, st_ta_endpoint)
|
| 126 |
logger.debug(f"analyzer_params: {analyzer_params}")
|
| 127 |
|
| 128 |
+
st_recognition_profile = st.sidebar.selectbox(
|
| 129 |
+
"Recognition profile",
|
| 130 |
+
["Dutch / EU", "General / International"],
|
| 131 |
+
index=0,
|
| 132 |
+
help=(
|
| 133 |
+
"Dutch / EU enables Dutch pattern recognizers such as BSN, postcode, "
|
| 134 |
+
"KvK, BTW/VAT, Dutch IBAN, Dutch phone numbers and Dutch license plates."
|
| 135 |
+
),
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
st_operator = st.sidebar.selectbox(
|
| 139 |
"De-identification approach",
|
| 140 |
["redact", "replace", "synthesize", "highlight", "mask", "hash", "encrypt"],
|
|
|
|
| 301 |
|
| 302 |
analyzer_load_state.empty()
|
| 303 |
|
| 304 |
+
if st_recognition_profile == "Dutch / EU":
|
| 305 |
+
st.info(
|
| 306 |
+
"Dutch / EU mode is active. The app adds Dutch pattern recognizers "
|
| 307 |
+
"for BSN, postcode, KvK, BTW/VAT, Dutch IBAN, Dutch phone numbers, "
|
| 308 |
+
"license plates, rijbewijs-style numbers and BIG numbers. Always review "
|
| 309 |
+
"the editable replacement table before exporting."
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
# Read default text
|
| 313 |
with open("demo_text.txt") as f:
|
| 314 |
demo_text = f.readlines()
|
|
|
|
| 346 |
|
| 347 |
try:
|
| 348 |
# Choose entities
|
| 349 |
+
all_supported_entities = list(get_supported_entities(*analyzer_params))
|
| 350 |
+
dutch_entities = set(get_dutch_entity_names())
|
| 351 |
+
|
| 352 |
+
if st_recognition_profile == "Dutch / EU":
|
| 353 |
+
preferred_entities = {
|
| 354 |
+
"PERSON",
|
| 355 |
+
"LOCATION",
|
| 356 |
+
"ORGANIZATION",
|
| 357 |
+
"EMAIL_ADDRESS",
|
| 358 |
+
"PHONE_NUMBER",
|
| 359 |
+
"IBAN_CODE",
|
| 360 |
+
"URL",
|
| 361 |
+
"IP_ADDRESS",
|
| 362 |
+
"GENERIC_PII",
|
| 363 |
+
} | dutch_entities
|
| 364 |
+
default_entities = [
|
| 365 |
+
entity for entity in all_supported_entities if entity in preferred_entities
|
| 366 |
+
]
|
| 367 |
+
else:
|
| 368 |
+
default_entities = list(all_supported_entities)
|
| 369 |
+
|
| 370 |
st_entities_expander = st.sidebar.expander("Choose entities to look for")
|
| 371 |
st_entities = st_entities_expander.multiselect(
|
| 372 |
label="Which entities to look for?",
|
| 373 |
+
options=all_supported_entities,
|
| 374 |
+
default=default_entities,
|
| 375 |
help="Limit the list of PII entities detected. "
|
| 376 |
+
"Dutch / EU mode adds recognizers such as NL_BSN, NL_POSTCODE, "
|
| 377 |
+
"NL_KVK_NUMBER, NL_VAT_NUMBER, NL_IBAN and NL_PHONE_NUMBER.",
|
| 378 |
)
|
| 379 |
|
| 380 |
# Before
|
|
|
|
| 382 |
analyzer = analyzer_engine(*analyzer_params)
|
| 383 |
analyzer_load_state.empty()
|
| 384 |
|
| 385 |
+
# The current demo uses English NER models. Dutch/EU pattern recognizers
|
| 386 |
+
# are registered under language="en" so they can run without requiring a
|
| 387 |
+
# separate Dutch NLP model.
|
| 388 |
st_analyze_results = analyze(
|
| 389 |
*analyzer_params,
|
| 390 |
text=st_text,
|