Upgraded OpenAI privacy-filter from linear 8 label token classifier to zero shot GLiNER PII filter.

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

gliner_config.json

@@ -0,0 +1,164 @@
+{
+  "blank_entity_prob": 0.1,
+  "class_token_index": 200020,
+  "decoder_loss_coef": 0.5,
+  "decoder_mode": "span",
+  "dropout": 0.2,
+  "embed_ent_token": true,
+  "encoder_config": {
+    "_name_or_path": "/workspace/privacy_filter_base_encoder",
+    "architectures": [
+      "OpenAIPrivacyFilterModel"
+    ],
+    "attention_bias": true,
+    "attention_dropout": 0.0,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "classifier_dropout": 0.0,
+    "default_n_ctx": 128000,
+    "dtype": "bfloat16",
+    "eos_token_id": 199999,
+    "head_dim": 64,
+    "hidden_act": "silu",
+    "hidden_size": 640,
+    "id2label": {
+      "0": "O",
+      "1": "B-account_number",
+      "2": "I-account_number",
+      "3": "E-account_number",
+      "4": "S-account_number",
+      "5": "B-private_address",
+      "6": "I-private_address",
+      "7": "E-private_address",
+      "8": "S-private_address",
+      "9": "B-private_date",
+      "10": "I-private_date",
+      "11": "E-private_date",
+      "12": "S-private_date",
+      "13": "B-private_email",
+      "14": "I-private_email",
+      "15": "E-private_email",
+      "16": "S-private_email",
+      "17": "B-private_person",
+      "18": "I-private_person",
+      "19": "E-private_person",
+      "20": "S-private_person",
+      "21": "B-private_phone",
+      "22": "I-private_phone",
+      "23": "E-private_phone",
+      "24": "S-private_phone",
+      "25": "B-private_url",
+      "26": "I-private_url",
+      "27": "E-private_url",
+      "28": "S-private_url",
+      "29": "B-secret",
+      "30": "I-secret",
+      "31": "E-secret",
+      "32": "S-secret"
+    },
+    "initial_context_length": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 640,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "B-account_number": 1,
+      "B-private_address": 5,
+      "B-private_date": 9,
+      "B-private_email": 13,
+      "B-private_person": 17,
+      "B-private_phone": 21,
+      "B-private_url": 25,
+      "B-secret": 29,
+      "E-account_number": 3,
+      "E-private_address": 7,
+      "E-private_date": 11,
+      "E-private_email": 15,
+      "E-private_person": 19,
+      "E-private_phone": 23,
+      "E-private_url": 27,
+      "E-secret": 31,
+      "I-account_number": 2,
+      "I-private_address": 6,
+      "I-private_date": 10,
+      "I-private_email": 14,
+      "I-private_person": 18,
+      "I-private_phone": 22,
+      "I-private_url": 26,
+      "I-secret": 30,
+      "O": 0,
+      "S-account_number": 4,
+      "S-private_address": 8,
+      "S-private_date": 12,
+      "S-private_email": 16,
+      "S-private_person": 20,
+      "S-private_phone": 24,
+      "S-private_url": 28,
+      "S-secret": 32
+    },
+    "max_position_embeddings": 131072,
+    "model_type": "openai_privacy_filter",
+    "num_attention_heads": 14,
+    "num_experts_per_tok": 4,
+    "num_hidden_layers": 8,
+    "num_key_value_heads": 2,
+    "num_local_experts": 128,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_router_logits": false,
+    "pad_token_id": 199999,
+    "problem_type": null,
+    "return_dict": true,
+    "rms_norm_eps": 1e-05,
+    "rope_parameters": {
+      "beta_fast": 32.0,
+      "beta_slow": 1.0,
+      "factor": 32.0,
+      "original_max_position_embeddings": 4096,
+      "rope_theta": 150000.0,
+      "rope_type": "yarn",
+      "truncate": false
+    },
+    "router_aux_loss_coef": 0.001,
+    "sliding_window": 128,
+    "tie_word_embeddings": false,
+    "transformers.js_config": {
+      "use_external_data_format": {
+        "model": 1,
+        "model.onnx": 3,
+        "model_fp16.onnx": 2
+      }
+    },
+    "use_cache": true,
+    "vocab_size": 200022
+  },
+  "ent_token": "<<ENT>>",
+  "eos_token_id": 199999,
+  "fine_tune": false,
+  "fuse_layers": false,
+  "hidden_size": 768,
+  "id_to_classes": null,
+  "labels_decoder": null,
+  "labels_encoder": null,
+  "max_len": 512,
+  "max_neg_type_ratio": 1,
+  "max_types": 100,
+  "max_width": 12,
+  "model_name": "/workspace/privacy_filter_base_encoder",
+  "model_type": null,
+  "name": "span level gliner",
+  "neg_spans_ratio": 1.0,
+  "num_post_fusion_layers": 1,
+  "num_rnn_layers": 1,
+  "pad_token_id": 199999,
+  "post_fusion_schema": null,
+  "precomputed_prompts_mode": null,
+  "represent_spans": false,
+  "sep_token": "<<SEP>>",
+  "span_loss_coef": 0.5,
+  "span_mode": "markerV0",
+  "subtoken_pooling": "first",
+  "token_loss_coef": 1.0,
+  "transformers_version": "5.7.0",
+  "vocab_size": 200022,
+  "words_splitter_type": "whitespace"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db448b8247e5442bdc7b663600dc3f99d97044b2bdb28c24abc52a13aaec0404
+size 2849509299

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59178ffd19a651cc414e9bc0a067207576b4f699ec96d2651a0cd513a6490de7
+size 27869768

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "backend": "tokenizers",
+  "eos_token": "<|endoftext|>",
+  "is_local": true,
+  "local_files_only": false,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 128000,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "TokenizersBackend"
+}