Upload native-weighted script-equitable tokenizer v0

MANIFEST.json

@@ -0,0 +1,64 @@
+{
+  "repo_id": "timothydillan/oim-script-equitable-regional-sp-bpe-32k-native-weighted-v0",
+  "tokenizer_name": "OIM Script-Equitable Regional SentencePiece BPE 32k Native-Weighted v0",
+  "tokenizer_kind": "sentencepiece",
+  "release_status": "private/research v0; source licenses not fully resolved",
+  "languages": [
+    "id",
+    "jv",
+    "su",
+    "ban",
+    "bug",
+    "ljp"
+  ],
+  "tags": [
+    "script-equitable",
+    "native-script",
+    "aksara",
+    "sentencepiece",
+    "native-weighted"
+  ],
+  "created_at": "2026-06-27T13:28:32+00:00",
+  "files": [
+    {
+      "path": "README.md",
+      "bytes": 7772,
+      "sha256": "6a816a7fe883a087fb2b42aa736698fb10f6b8f09b5891e7f14389be7f13a13b"
+    },
+    {
+      "path": "benchmarks/tokenizer_benchmark_script_equitable_v0_by_native_script.json",
+      "bytes": 33994,
+      "sha256": "4f242d36b739e093f2e754e4d2f43b4aafc4f565c31af2768af213e2b485b8ab"
+    },
+    {
+      "path": "benchmarks/tokenizer_benchmark_script_equitable_v0_latn_weighted_compare.json",
+      "bytes": 4500,
+      "sha256": "a4ea2069b9dd28d4c770df73261ebf073d1d8554114da820e610705628488726"
+    },
+    {
+      "path": "benchmarks/tokenizer_benchmark_script_equitable_v0_native_weighted_compare.json",
+      "bytes": 4478,
+      "sha256": "d2fd0d36fd03c7585dbe022c61ed1269a619102174930c7695ba69afe9f3015e"
+    },
+    {
+      "path": "benchmarks/tokenizer_benchmark_script_equitable_v0_weighted_compare.json",
+      "bytes": 4535,
+      "sha256": "df6797e38ac04483b61bd62aa8e8f17286d0463d441f6ab97b78036cb82b849d"
+    },
+    {
+      "path": "oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe.model",
+      "bytes": 756461,
+      "sha256": "b70f7397cff2d249fe4857d5edbdadfd281c119c09d70a5f2358b37d49f3402b"
+    },
+    {
+      "path": "oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe.vocab",
+      "bytes": 472974,
+      "sha256": "2d05e924767f7a04f85773ff9117fba3a8b751227a75b59af1a7ee0fba992890"
+    },
+    {
+      "path": "provenance/script_equitable_v0_native_weighted_manifest.json",
+      "bytes": 5269,
+      "sha256": "c43c93073572356cdfd67280fa6bca00c23096439f113981e0c0ef857f9f2872"
+    }
+  ]
+}

README.md

@@ -0,0 +1,126 @@
+---
+library_name: tokenizers
+language:
+- id
+- jv
+- su
+- ban
+- bug
+- ljp
+tags:
+- tokenizer
+- indonesian
+- regional-languages
+- low-resource-languages
+- sentencepiece
+- script-equitable
+- native-script
+- aksara
+- native-weighted
+license: other
+---
+
+# OIM Script-Equitable Regional SentencePiece BPE 32k Native-Weighted v0
+
+This is a draft Open Indonesia Models tokenizer artifact for Indonesian and
+regional-language tokenizer research.
+
+Repository target: `timothydillan/oim-script-equitable-regional-sp-bpe-32k-native-weighted-v0`
+Release status: **private/research v0; source licenses not fully resolved**
+
+## Corpus
+
+Tokenizer candidate trained on a 130,969-row native-weighted research corpus built from CC-100 Indonesian/Javanese/Sundanese Latin samples, Wikipedia Indonesian/Balinese/Javanese/Sundanese samples, NusaAksara native-script transcriptions, NusaAksara Latin transliterations, and Balinese research SFT message content. Native NusaAksara rows are upweighted 8x and transliterations 2x to prevent Latin corpora from dominating merge selection. Intended as a private/research tokenizer artifact and audit tool, not as a drop-in tokenizer replacement for existing pretrained LLMs.
+
+## Metrics
+
+Lower `tokens/word` is better.
+
+| language | tokenizer | tokens/word | chars/token | fragmented words |
+|---|---:|---:|---:|---:|
+| script-equitable-v0-heldout | xlm-roberta-base | 1.797 | 3.878 | 42.6% |
+| script-equitable-v0-heldout | tiktoken/cl100k_base | 2.409 | 2.892 | 80.5% |
+| script-equitable-v0-heldout | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 1.930 | 3.609 | 44.9% |
+| script-equitable-v0-heldout | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 1.513 | 4.603 | 21.6% |
+| script-equitable-v0-heldout | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 1.526 | 4.567 | 22.4% |
+| script-equitable-v0-latn-heldout | xlm-roberta-base | 1.803 | 3.883 | 42.3% |
+| script-equitable-v0-latn-heldout | tiktoken/cl100k_base | 2.377 | 2.946 | 80.4% |
+| script-equitable-v0-latn-heldout | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 1.939 | 3.612 | 44.7% |
+| script-equitable-v0-latn-heldout | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 1.511 | 4.636 | 21.3% |
+| script-equitable-v0-latn-heldout | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 1.527 | 4.585 | 22.2% |
+| script-equitable-v0-native-heldout | xlm-roberta-base | 1.028 | 2.816 | 79.0% |
+| script-equitable-v0-native-heldout | tiktoken/cl100k_base | 5.972 | 0.485 | 82.6% |
+| script-equitable-v0-native-heldout | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 0.927 | 3.123 | 62.4% |
+| script-equitable-v0-native-heldout | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 1.829 | 1.583 | 51.7% |
+| script-equitable-v0-native-heldout | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 1.313 | 2.206 | 36.9% |
+| native-script/bali | xlm-roberta-base | 0.120 | 21.750 | 100.0% |
+| native-script/bali | tiktoken/cl100k_base | 7.671 | 0.340 | 100.0% |
+| native-script/bali | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 0.948 | 2.754 | 62.2% |
+| native-script/bali | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 1.649 | 1.584 | 60.1% |
+| native-script/bali | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 1.195 | 2.186 | 55.0% |
+| native-script/batak | xlm-roberta-base | 0.604 | 3.956 | 100.0% |
+| native-script/batak | tiktoken/cl100k_base | 7.118 | 0.336 | 100.0% |
+| native-script/batak | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 0.604 | 3.956 | 100.0% |
+| native-script/batak | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 2.032 | 1.176 | 55.6% |
+| native-script/batak | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 1.492 | 1.602 | 26.2% |
+| native-script/jawa | xlm-roberta-base | 0.233 | 11.439 | 99.9% |
+| native-script/jawa | tiktoken/cl100k_base | 7.827 | 0.340 | 99.9% |
+| native-script/jawa | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 0.233 | 11.439 | 99.9% |
+| native-script/jawa | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 1.774 | 1.500 | 69.1% |
+| native-script/jawa | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 1.296 | 2.053 | 36.7% |
+| native-script/jawi | xlm-roberta-base | 2.074 | 2.316 | 62.4% |
+| native-script/jawi | tiktoken/cl100k_base | 3.720 | 1.291 | 93.1% |
+| native-script/jawi | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 1.658 | 2.896 | 36.6% |
+| native-script/jawi | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 2.909 | 1.651 | 74.7% |
+| native-script/jawi | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 2.017 | 2.381 | 48.9% |
+| native-script/lampung | xlm-roberta-base | 3.442 | 1.473 | 88.5% |
+| native-script/lampung | tiktoken/cl100k_base | 3.035 | 1.671 | 90.3% |
+| native-script/lampung | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 3.053 | 1.661 | 88.5% |
+| native-script/lampung | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 3.088 | 1.642 | 87.6% |
+| native-script/lampung | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 2.779 | 1.825 | 83.2% |
+| native-script/lontara | xlm-roberta-base | 0.872 | 3.144 | 97.5% |
+| native-script/lontara | tiktoken/cl100k_base | 6.933 | 0.395 | 100.0% |
+| native-script/lontara | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 1.067 | 2.570 | 49.0% |
+| native-script/lontara | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 1.665 | 1.646 | 57.9% |
+| native-script/lontara | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 1.121 | 2.446 | 46.3% |
+| native-script/pegon | xlm-roberta-base | 2.020 | 1.202 | 15.9% |
+| native-script/pegon | tiktoken/cl100k_base | 2.287 | 1.061 | 17.8% |
+| native-script/pegon | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 0.747 | 3.251 | 10.5% |
+| native-script/pegon | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 1.310 | 1.852 | 20.4% |
+| native-script/pegon | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 0.878 | 2.763 | 11.6% |
+| native-script/sunda | xlm-roberta-base | 1.166 | 2.477 | 100.0% |
+| native-script/sunda | tiktoken/cl100k_base | 7.253 | 0.398 | 100.0% |
+| native-script/sunda | local-sentencepiece/oim-script-equitable-regional-sp-bpe | 1.166 | 2.477 | 100.0% |
+| native-script/sunda | local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe | 2.022 | 1.428 | 32.8% |
+| native-script/sunda | local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe | 1.473 | 1.961 | 24.1% |
+
+## Intended Use
+
+- tokenizer fertility audits;
+- retrieval and chunking experiments for Indonesian regional-language corpora;
+- future vocabulary-extension or from-scratch tokenizer research.
+
+Do not treat this as a drop-in replacement for a pretrained LLM tokenizer unless
+the model is trained or continued-pretrained with this tokenizer.
+
+## Release Notes
+
+This artifact is suitable for private/research publication only until source
+licenses and redistribution terms are resolved. It contains tokenizer model
+files, not raw corpus rows.
+
+## Files
+
+- `MANIFEST.json`
+- `benchmarks/*.json`
+- `provenance/*.json`
+- `tokenizer.json`
+- `*vocab.json`
+- `*merges.txt`
+- `*.model` / `*.vocab` for SentencePiece tokenizers
+
+## Upload
+
+```bash
+huggingface-cli upload timothydillan/oim-script-equitable-regional-sp-bpe-32k-native-weighted-v0 . --repo-type model
+```

benchmarks/tokenizer_benchmark_script_equitable_v0_by_native_script.json

@@ -0,0 +1,1122 @@
+[
+  {
+    "language": "native-script/bali",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 46,
+    "char_count": 2523,
+    "byte_count": 7545,
+    "word_count": 966,
+    "token_count": 116,
+    "tokens_per_word": 0.12008281573498965,
+    "tokens_per_char": 0.04597701149425287,
+    "chars_per_token": 21.75,
+    "bytes_per_token": 65.04310344827586,
+    "word_fragmentation_rate": 1.0,
+    "mean_tokens_per_fragmented_word": 2.0,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 2511,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 3,
+      "native_script_chars": 2511
+    }
+  },
+  {
+    "language": "native-script/bali",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 46,
+    "char_count": 2523,
+    "byte_count": 7545,
+    "word_count": 966,
+    "token_count": 7410,
+    "tokens_per_word": 7.670807453416149,
+    "tokens_per_char": 2.9369797859690845,
+    "chars_per_token": 0.34048582995951415,
+    "bytes_per_token": 1.0182186234817814,
+    "word_fragmentation_rate": 1.0,
+    "mean_tokens_per_fragmented_word": 4.418219461697722,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 2511,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 3,
+      "native_script_chars": 2511
+    }
+  },
+  {
+    "language": "native-script/bali",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 46,
+    "char_count": 2523,
+    "byte_count": 7545,
+    "word_count": 966,
+    "token_count": 916,
+    "tokens_per_word": 0.94824016563147,
+    "tokens_per_char": 0.363059849385652,
+    "chars_per_token": 2.754366812227074,
+    "bytes_per_token": 8.236899563318778,
+    "word_fragmentation_rate": 0.6221532091097308,
+    "mean_tokens_per_fragmented_word": 2.2778702163061566,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 2511,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 3,
+      "native_script_chars": 2511
+    }
+  },
+  {
+    "language": "native-script/bali",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 46,
+    "char_count": 2523,
+    "byte_count": 7545,
+    "word_count": 966,
+    "token_count": 1593,
+    "tokens_per_word": 1.6490683229813665,
+    "tokens_per_char": 0.6313912009512486,
+    "chars_per_token": 1.5838041431261771,
+    "bytes_per_token": 4.736346516007533,
+    "word_fragmentation_rate": 0.6014492753623188,
+    "mean_tokens_per_fragmented_word": 2.3648881239242683,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 2511,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 3,
+      "native_script_chars": 2511
+    }
+  },
+  {
+    "language": "native-script/bali",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 46,
+    "char_count": 2523,
+    "byte_count": 7545,
+    "word_count": 966,
+    "token_count": 1154,
+    "tokens_per_word": 1.194616977225673,
+    "tokens_per_char": 0.45739199365834327,
+    "chars_per_token": 2.18630849220104,
+    "bytes_per_token": 6.538128249566724,
+    "word_fragmentation_rate": 0.5496894409937888,
+    "mean_tokens_per_fragmented_word": 2.3333333333333335,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 2511,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 3,
+      "native_script_chars": 2511
+    }
+  },
+  {
+    "language": "native-script/batak",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 53,
+    "char_count": 447,
+    "byte_count": 1333,
+    "word_count": 187,
+    "token_count": 113,
+    "tokens_per_word": 0.6042780748663101,
+    "tokens_per_char": 0.25279642058165547,
+    "chars_per_token": 3.9557522123893807,
+    "bytes_per_token": 11.79646017699115,
+    "word_fragmentation_rate": 1.0,
+    "mean_tokens_per_fragmented_word": 2.0,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 443,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 2,
+      "native_script_chars": 443
+    }
+  },
+  {
+    "language": "native-script/batak",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 53,
+    "char_count": 447,
+    "byte_count": 1333,
+    "word_count": 187,
+    "token_count": 1331,
+    "tokens_per_word": 7.117647058823529,
+    "tokens_per_char": 2.9776286353467563,
+    "chars_per_token": 0.33583771600300527,
+    "bytes_per_token": 1.0015026296018033,
+    "word_fragmentation_rate": 1.0,
+    "mean_tokens_per_fragmented_word": 4.0588235294117645,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 443,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 2,
+      "native_script_chars": 443
+    }
+  },
+  {
+    "language": "native-script/batak",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 53,
+    "char_count": 447,
+    "byte_count": 1333,
+    "word_count": 187,
+    "token_count": 113,
+    "tokens_per_word": 0.6042780748663101,
+    "tokens_per_char": 0.25279642058165547,
+    "chars_per_token": 3.9557522123893807,
+    "bytes_per_token": 11.79646017699115,
+    "word_fragmentation_rate": 1.0,
+    "mean_tokens_per_fragmented_word": 2.0,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 443,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 2,
+      "native_script_chars": 443
+    }
+  },
+  {
+    "language": "native-script/batak",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 53,
+    "char_count": 447,
+    "byte_count": 1333,
+    "word_count": 187,
+    "token_count": 380,
+    "tokens_per_word": 2.0320855614973263,
+    "tokens_per_char": 0.8501118568232662,
+    "chars_per_token": 1.1763157894736842,
+    "bytes_per_token": 3.5078947368421054,
+    "word_fragmentation_rate": 0.5561497326203209,
+    "mean_tokens_per_fragmented_word": 2.375,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 443,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 2,
+      "native_script_chars": 443
+    }
+  },
+  {
+    "language": "native-script/batak",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 53,
+    "char_count": 447,
+    "byte_count": 1333,
+    "word_count": 187,
+    "token_count": 279,
+    "tokens_per_word": 1.4919786096256684,
+    "tokens_per_char": 0.6241610738255033,
+    "chars_per_token": 1.6021505376344085,
+    "bytes_per_token": 4.777777777777778,
+    "word_fragmentation_rate": 0.2620320855614973,
+    "mean_tokens_per_fragmented_word": 2.326530612244898,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 443,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 2,
+      "native_script_chars": 443
+    }
+  },
+  {
+    "language": "native-script/jawa",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 69,
+    "char_count": 2059,
+    "byte_count": 6117,
+    "word_count": 774,
+    "token_count": 180,
+    "tokens_per_word": 0.23255813953488372,
+    "tokens_per_char": 0.08742107819329771,
+    "chars_per_token": 11.438888888888888,
+    "bytes_per_token": 33.983333333333334,
+    "word_fragmentation_rate": 0.9987080103359173,
+    "mean_tokens_per_fragmented_word": 2.0,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 2026,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 12,
+      "native_script_chars": 2026
+    }
+  },
+  {
+    "language": "native-script/jawa",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 69,
+    "char_count": 2059,
+    "byte_count": 6117,
+    "word_count": 774,
+    "token_count": 6058,
+    "tokens_per_word": 7.82687338501292,
+    "tokens_per_char": 2.9422049538610975,
+    "chars_per_token": 0.33988114889402443,
+    "bytes_per_token": 1.0097391878507758,
+    "word_fragmentation_rate": 0.9987080103359173,
+    "mean_tokens_per_fragmented_word": 4.199223803363519,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 2026,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 12,
+      "native_script_chars": 2026
+    }
+  },
+  {
+    "language": "native-script/jawa",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 69,
+    "char_count": 2059,
+    "byte_count": 6117,
+    "word_count": 774,
+    "token_count": 180,
+    "tokens_per_word": 0.23255813953488372,
+    "tokens_per_char": 0.08742107819329771,
+    "chars_per_token": 11.438888888888888,
+    "bytes_per_token": 33.983333333333334,
+    "word_fragmentation_rate": 0.9987080103359173,
+    "mean_tokens_per_fragmented_word": 2.0,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 2026,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 12,
+      "native_script_chars": 2026
+    }
+  },
+  {
+    "language": "native-script/jawa",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 69,
+    "char_count": 2059,
+    "byte_count": 6117,
+    "word_count": 774,
+    "token_count": 1373,
+    "tokens_per_word": 1.7739018087855296,
+    "tokens_per_char": 0.6668285575522098,
+    "chars_per_token": 1.4996358339402767,
+    "bytes_per_token": 4.455207574654042,
+    "word_fragmentation_rate": 0.6912144702842378,
+    "mean_tokens_per_fragmented_word": 2.3607476635514018,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 2026,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 12,
+      "native_script_chars": 2026
+    }
+  },
+  {
+    "language": "native-script/jawa",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 69,
+    "char_count": 2059,
+    "byte_count": 6117,
+    "word_count": 774,
+    "token_count": 1003,
+    "tokens_per_word": 1.2958656330749354,
+    "tokens_per_char": 0.48712967459932005,
+    "chars_per_token": 2.0528414755732802,
+    "bytes_per_token": 6.098703888334995,
+    "word_fragmentation_rate": 0.3669250645994832,
+    "mean_tokens_per_fragmented_word": 2.267605633802817,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 2026,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 12,
+      "native_script_chars": 2026
+    }
+  },
+  {
+    "language": "native-script/jawi",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 65,
+    "char_count": 1955,
+    "byte_count": 3537,
+    "word_count": 407,
+    "token_count": 844,
+    "tokens_per_word": 2.0737100737100738,
+    "tokens_per_char": 0.43171355498721226,
+    "chars_per_token": 2.316350710900474,
+    "bytes_per_token": 4.190758293838862,
+    "word_fragmentation_rate": 0.6240786240786241,
+    "mean_tokens_per_fragmented_word": 2.5669291338582676,
+    "script_counts": {
+      "latin_chars": 12,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1566,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 23,
+      "native_script_chars": 1568
+    }
+  },
+  {
+    "language": "native-script/jawi",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 65,
+    "char_count": 1955,
+    "byte_count": 3537,
+    "word_count": 407,
+    "token_count": 1514,
+    "tokens_per_word": 3.7199017199017197,
+    "tokens_per_char": 0.7744245524296676,
+    "chars_per_token": 1.2912813738441216,
+    "bytes_per_token": 2.336195508586526,
+    "word_fragmentation_rate": 0.9312039312039312,
+    "mean_tokens_per_fragmented_word": 3.7493403693931397,
+    "script_counts": {
+      "latin_chars": 12,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1566,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 23,
+      "native_script_chars": 1568
+    }
+  },
+  {
+    "language": "native-script/jawi",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 65,
+    "char_count": 1955,
+    "byte_count": 3537,
+    "word_count": 407,
+    "token_count": 675,
+    "tokens_per_word": 1.6584766584766584,
+    "tokens_per_char": 0.3452685421994885,
+    "chars_per_token": 2.8962962962962964,
+    "bytes_per_token": 5.24,
+    "word_fragmentation_rate": 0.36609336609336607,
+    "mean_tokens_per_fragmented_word": 2.61744966442953,
+    "script_counts": {
+      "latin_chars": 12,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1566,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 23,
+      "native_script_chars": 1568
+    }
+  },
+  {
+    "language": "native-script/jawi",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 65,
+    "char_count": 1955,
+    "byte_count": 3537,
+    "word_count": 407,
+    "token_count": 1184,
+    "tokens_per_word": 2.909090909090909,
+    "tokens_per_char": 0.6056265984654732,
+    "chars_per_token": 1.6511824324324325,
+    "bytes_per_token": 2.987331081081081,
+    "word_fragmentation_rate": 0.7469287469287469,
+    "mean_tokens_per_fragmented_word": 3.4375,
+    "script_counts": {
+      "latin_chars": 12,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1566,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 23,
+      "native_script_chars": 1568
+    }
+  },
+  {
+    "language": "native-script/jawi",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 65,
+    "char_count": 1955,
+    "byte_count": 3537,
+    "word_count": 407,
+    "token_count": 821,
+    "tokens_per_word": 2.017199017199017,
+    "tokens_per_char": 0.41994884910485936,
+    "chars_per_token": 2.3812423873325215,
+    "bytes_per_token": 4.3081607795371495,
+    "word_fragmentation_rate": 0.48894348894348894,
+    "mean_tokens_per_fragmented_word": 2.9246231155778895,
+    "script_counts": {
+      "latin_chars": 12,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1566,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 23,
+      "native_script_chars": 1568
+    }
+  },
+  {
+    "language": "native-script/lampung",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 85,
+    "char_count": 573,
+    "byte_count": 573,
+    "word_count": 113,
+    "token_count": 389,
+    "tokens_per_word": 3.4424778761061945,
+    "tokens_per_char": 0.6788830715532286,
+    "chars_per_token": 1.4730077120822622,
+    "bytes_per_token": 1.4730077120822622,
+    "word_fragmentation_rate": 0.8849557522123894,
+    "mean_tokens_per_fragmented_word": 2.78,
+    "script_counts": {
+      "latin_chars": 461,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 100,
+      "native_script_chars": 0
+    }
+  },
+  {
+    "language": "native-script/lampung",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 85,
+    "char_count": 573,
+    "byte_count": 573,
+    "word_count": 113,
+    "token_count": 343,
+    "tokens_per_word": 3.0353982300884956,
+    "tokens_per_char": 0.5986038394415357,
+    "chars_per_token": 1.6705539358600583,
+    "bytes_per_token": 1.6705539358600583,
+    "word_fragmentation_rate": 0.9026548672566371,
+    "mean_tokens_per_fragmented_word": 2.764705882352941,
+    "script_counts": {
+      "latin_chars": 461,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 100,
+      "native_script_chars": 0
+    }
+  },
+  {
+    "language": "native-script/lampung",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 85,
+    "char_count": 573,
+    "byte_count": 573,
+    "word_count": 113,
+    "token_count": 345,
+    "tokens_per_word": 3.0530973451327434,
+    "tokens_per_char": 0.6020942408376964,
+    "chars_per_token": 1.6608695652173913,
+    "bytes_per_token": 1.6608695652173913,
+    "word_fragmentation_rate": 0.8849557522123894,
+    "mean_tokens_per_fragmented_word": 2.81,
+    "script_counts": {
+      "latin_chars": 461,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 100,
+      "native_script_chars": 0
+    }
+  },
+  {
+    "language": "native-script/lampung",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 85,
+    "char_count": 573,
+    "byte_count": 573,
+    "word_count": 113,
+    "token_count": 349,
+    "tokens_per_word": 3.088495575221239,
+    "tokens_per_char": 0.6090750436300174,
+    "chars_per_token": 1.6418338108882522,
+    "bytes_per_token": 1.6418338108882522,
+    "word_fragmentation_rate": 0.8761061946902655,
+    "mean_tokens_per_fragmented_word": 2.878787878787879,
+    "script_counts": {
+      "latin_chars": 461,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 100,
+      "native_script_chars": 0
+    }
+  },
+  {
+    "language": "native-script/lampung",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 85,
+    "char_count": 573,
+    "byte_count": 573,
+    "word_count": 113,
+    "token_count": 314,
+    "tokens_per_word": 2.7787610619469025,
+    "tokens_per_char": 0.5479930191972077,
+    "chars_per_token": 1.8248407643312101,
+    "bytes_per_token": 1.8248407643312101,
+    "word_fragmentation_rate": 0.831858407079646,
+    "mean_tokens_per_fragmented_word": 2.617021276595745,
+    "script_counts": {
+      "latin_chars": 461,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 100,
+      "native_script_chars": 0
+    }
+  },
+  {
+    "language": "native-script/lontara",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 39,
+    "char_count": 1113,
+    "byte_count": 3002,
+    "word_count": 406,
+    "token_count": 354,
+    "tokens_per_word": 0.8719211822660099,
+    "tokens_per_char": 0.31805929919137466,
+    "chars_per_token": 3.1440677966101696,
+    "bytes_per_token": 8.480225988700566,
+    "word_fragmentation_rate": 0.9753694581280788,
+    "mean_tokens_per_fragmented_word": 2.0,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 925,
+      "batak_chars": 0,
+      "arabic_based_chars": 39,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 7,
+      "native_script_chars": 964
+    }
+  },
+  {
+    "language": "native-script/lontara",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 39,
+    "char_count": 1113,
+    "byte_count": 3002,
+    "word_count": 406,
+    "token_count": 2815,
+    "tokens_per_word": 6.933497536945813,
+    "tokens_per_char": 2.5292003593890384,
+    "chars_per_token": 0.39538188277087033,
+    "bytes_per_token": 1.066429840142096,
+    "word_fragmentation_rate": 1.0,
+    "mean_tokens_per_fragmented_word": 4.440886699507389,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 925,
+      "batak_chars": 0,
+      "arabic_based_chars": 39,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 7,
+      "native_script_chars": 964
+    }
+  },
+  {
+    "language": "native-script/lontara",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 39,
+    "char_count": 1113,
+    "byte_count": 3002,
+    "word_count": 406,
+    "token_count": 433,
+    "tokens_per_word": 1.0665024630541873,
+    "tokens_per_char": 0.3890386343216532,
+    "chars_per_token": 2.5704387990762125,
+    "bytes_per_token": 6.933025404157044,
+    "word_fragmentation_rate": 0.49014778325123154,
+    "mean_tokens_per_fragmented_word": 2.1959798994974875,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 925,
+      "batak_chars": 0,
+      "arabic_based_chars": 39,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 7,
+      "native_script_chars": 964
+    }
+  },
+  {
+    "language": "native-script/lontara",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 39,
+    "char_count": 1113,
+    "byte_count": 3002,
+    "word_count": 406,
+    "token_count": 676,
+    "tokens_per_word": 1.6650246305418719,
+    "tokens_per_char": 0.6073674752920036,
+    "chars_per_token": 1.6464497041420119,
+    "bytes_per_token": 4.440828402366864,
+    "word_fragmentation_rate": 0.5788177339901478,
+    "mean_tokens_per_fragmented_word": 2.3106382978723405,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 925,
+      "batak_chars": 0,
+      "arabic_based_chars": 39,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 7,
+      "native_script_chars": 964
+    }
+  },
+  {
+    "language": "native-script/lontara",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 39,
+    "char_count": 1113,
+    "byte_count": 3002,
+    "word_count": 406,
+    "token_count": 455,
+    "tokens_per_word": 1.1206896551724137,
+    "tokens_per_char": 0.4088050314465409,
+    "chars_per_token": 2.4461538461538463,
+    "bytes_per_token": 6.5978021978021975,
+    "word_fragmentation_rate": 0.4630541871921182,
+    "mean_tokens_per_fragmented_word": 2.175531914893617,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 925,
+      "batak_chars": 0,
+      "arabic_based_chars": 39,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 7,
+      "native_script_chars": 964
+    }
+  },
+  {
+    "language": "native-script/pegon",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 70,
+    "char_count": 2097,
+    "byte_count": 3985,
+    "word_count": 864,
+    "token_count": 1745,
+    "tokens_per_word": 2.019675925925926,
+    "tokens_per_char": 0.832141154029566,
+    "chars_per_token": 1.2017191977077364,
+    "bytes_per_token": 2.2836676217765044,
+    "word_fragmentation_rate": 0.15856481481481483,
+    "mean_tokens_per_fragmented_word": 2.1386861313868613,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1886,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 11,
+      "native_script_chars": 1886
+    }
+  },
+  {
+    "language": "native-script/pegon",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 70,
+    "char_count": 2097,
+    "byte_count": 3985,
+    "word_count": 864,
+    "token_count": 1976,
+    "tokens_per_word": 2.287037037037037,
+    "tokens_per_char": 0.9422985216976634,
+    "chars_per_token": 1.0612348178137652,
+    "bytes_per_token": 2.0167004048582995,
+    "word_fragmentation_rate": 0.17824074074074073,
+    "mean_tokens_per_fragmented_word": 2.409090909090909,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1886,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 11,
+      "native_script_chars": 1886
+    }
+  },
+  {
+    "language": "native-script/pegon",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 70,
+    "char_count": 2097,
+    "byte_count": 3985,
+    "word_count": 864,
+    "token_count": 645,
+    "tokens_per_word": 0.7465277777777778,
+    "tokens_per_char": 0.30758226037195996,
+    "chars_per_token": 3.2511627906976743,
+    "bytes_per_token": 6.178294573643411,
+    "word_fragmentation_rate": 0.10532407407407407,
+    "mean_tokens_per_fragmented_word": 2.142857142857143,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1886,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 11,
+      "native_script_chars": 1886
+    }
+  },
+  {
+    "language": "native-script/pegon",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 70,
+    "char_count": 2097,
+    "byte_count": 3985,
+    "word_count": 864,
+    "token_count": 1132,
+    "tokens_per_word": 1.3101851851851851,
+    "tokens_per_char": 0.5398187887458273,
+    "chars_per_token": 1.8524734982332156,
+    "bytes_per_token": 3.5203180212014136,
+    "word_fragmentation_rate": 0.2037037037037037,
+    "mean_tokens_per_fragmented_word": 2.1931818181818183,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1886,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 11,
+      "native_script_chars": 1886
+    }
+  },
+  {
+    "language": "native-script/pegon",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 70,
+    "char_count": 2097,
+    "byte_count": 3985,
+    "word_count": 864,
+    "token_count": 759,
+    "tokens_per_word": 0.8784722222222222,
+    "tokens_per_char": 0.36194563662374823,
+    "chars_per_token": 2.7628458498023716,
+    "bytes_per_token": 5.250329380764163,
+    "word_fragmentation_rate": 0.11574074074074074,
+    "mean_tokens_per_fragmented_word": 2.17,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 0,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 1886,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 11,
+      "native_script_chars": 1886
+    }
+  },
+  {
+    "language": "native-script/sunda",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 73,
+    "char_count": 1692,
+    "byte_count": 4488,
+    "word_count": 586,
+    "token_count": 683,
+    "tokens_per_word": 1.1655290102389078,
+    "tokens_per_char": 0.4036643026004728,
+    "chars_per_token": 2.4773060029282576,
+    "bytes_per_token": 6.571010248901904,
+    "word_fragmentation_rate": 1.0,
+    "mean_tokens_per_fragmented_word": 2.006825938566553,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 1397,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 61,
+      "native_script_chars": 1397
+    }
+  },
+  {
+    "language": "native-script/sunda",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 73,
+    "char_count": 1692,
+    "byte_count": 4488,
+    "word_count": 586,
+    "token_count": 4250,
+    "tokens_per_word": 7.252559726962457,
+    "tokens_per_char": 2.5118203309692673,
+    "chars_per_token": 0.3981176470588235,
+    "bytes_per_token": 1.056,
+    "word_fragmentation_rate": 1.0,
+    "mean_tokens_per_fragmented_word": 4.242320819112628,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 1397,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 61,
+      "native_script_chars": 1397
+    }
+  },
+  {
+    "language": "native-script/sunda",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 73,
+    "char_count": 1692,
+    "byte_count": 4488,
+    "word_count": 586,
+    "token_count": 683,
+    "tokens_per_word": 1.1655290102389078,
+    "tokens_per_char": 0.4036643026004728,
+    "chars_per_token": 2.4773060029282576,
+    "bytes_per_token": 6.571010248901904,
+    "word_fragmentation_rate": 1.0,
+    "mean_tokens_per_fragmented_word": 2.006825938566553,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 1397,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 61,
+      "native_script_chars": 1397
+    }
+  },
+  {
+    "language": "native-script/sunda",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 73,
+    "char_count": 1692,
+    "byte_count": 4488,
+    "word_count": 586,
+    "token_count": 1185,
+    "tokens_per_word": 2.022184300341297,
+    "tokens_per_char": 0.700354609929078,
+    "chars_per_token": 1.4278481012658228,
+    "bytes_per_token": 3.787341772151899,
+    "word_fragmentation_rate": 0.32764505119453924,
+    "mean_tokens_per_fragmented_word": 2.3854166666666665,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 1397,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 61,
+      "native_script_chars": 1397
+    }
+  },
+  {
+    "language": "native-script/sunda",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 73,
+    "char_count": 1692,
+    "byte_count": 4488,
+    "word_count": 586,
+    "token_count": 863,
+    "tokens_per_word": 1.4726962457337884,
+    "tokens_per_char": 0.5100472813238771,
+    "chars_per_token": 1.9606025492468135,
+    "bytes_per_token": 5.200463499420626,
+    "word_fragmentation_rate": 0.24061433447098976,
+    "mean_tokens_per_fragmented_word": 2.326241134751773,
+    "script_counts": {
+      "latin_chars": 0,
+      "balinese_chars": 0,
+      "javanese_chars": 0,
+      "sundanese_chars": 1397,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 0,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 0,
+      "other_chars": 61,
+      "native_script_chars": 1397
+    }
+  }
+]

benchmarks/tokenizer_benchmark_script_equitable_v0_latn_weighted_compare.json

@@ -0,0 +1,142 @@
+[
+  {
+    "language": "script-equitable-v0-latn-heldout",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 9500,
+    "char_count": 3380271,
+    "byte_count": 3425301,
+    "word_count": 482692,
+    "token_count": 870482,
+    "tokens_per_word": 1.8033901535554764,
+    "tokens_per_char": 0.25751840606862586,
+    "chars_per_token": 3.8832175737120354,
+    "bytes_per_token": 3.9349475348140457,
+    "word_fragmentation_rate": 0.4225551697562835,
+    "mean_tokens_per_fragmented_word": 2.4725098546802378,
+    "script_counts": {
+      "latin_chars": 2756382,
+      "balinese_chars": 1271,
+      "javanese_chars": 136,
+      "sundanese_chars": 9,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 625,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 1,
+      "other_chars": 107438,
+      "native_script_chars": 2042
+    }
+  },
+  {
+    "language": "script-equitable-v0-latn-heldout",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 9500,
+    "char_count": 3380271,
+    "byte_count": 3425301,
+    "word_count": 482692,
+    "token_count": 1147322,
+    "tokens_per_word": 2.3769235868835614,
+    "tokens_per_char": 0.3394171650734512,
+    "chars_per_token": 2.9462269528519456,
+    "bytes_per_token": 2.985474871047535,
+    "word_fragmentation_rate": 0.8044405127907651,
+    "mean_tokens_per_fragmented_word": 2.682330793181508,
+    "script_counts": {
+      "latin_chars": 2756382,
+      "balinese_chars": 1271,
+      "javanese_chars": 136,
+      "sundanese_chars": 9,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 625,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 1,
+      "other_chars": 107438,
+      "native_script_chars": 2042
+    }
+  },
+  {
+    "language": "script-equitable-v0-latn-heldout",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 9500,
+    "char_count": 3380271,
+    "byte_count": 3425301,
+    "word_count": 482692,
+    "token_count": 935966,
+    "tokens_per_word": 1.939054303779636,
+    "tokens_per_char": 0.2768908173338765,
+    "chars_per_token": 3.61153182914764,
+    "bytes_per_token": 3.65964255111831,
+    "word_fragmentation_rate": 0.44694339247387566,
+    "mean_tokens_per_fragmented_word": 2.688355211925687,
+    "script_counts": {
+      "latin_chars": 2756382,
+      "balinese_chars": 1271,
+      "javanese_chars": 136,
+      "sundanese_chars": 9,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 625,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 1,
+      "other_chars": 107438,
+      "native_script_chars": 2042
+    }
+  },
+  {
+    "language": "script-equitable-v0-latn-heldout",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 9500,
+    "char_count": 3380271,
+    "byte_count": 3425301,
+    "word_count": 482692,
+    "token_count": 729131,
+    "tokens_per_word": 1.5105512417856521,
+    "tokens_per_char": 0.21570193632404028,
+    "chars_per_token": 4.636026996520515,
+    "bytes_per_token": 4.697785445962386,
+    "word_fragmentation_rate": 0.21337623163425123,
+    "mean_tokens_per_fragmented_word": 2.550657798922278,
+    "script_counts": {
+      "latin_chars": 2756382,
+      "balinese_chars": 1271,
+      "javanese_chars": 136,
+      "sundanese_chars": 9,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 625,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 1,
+      "other_chars": 107438,
+      "native_script_chars": 2042
+    }
+  },
+  {
+    "language": "script-equitable-v0-latn-heldout",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 9500,
+    "char_count": 3380271,
+    "byte_count": 3425301,
+    "word_count": 482692,
+    "token_count": 737265,
+    "tokens_per_word": 1.5274025672685687,
+    "tokens_per_char": 0.21810825226734779,
+    "chars_per_token": 4.584879249659213,
+    "bytes_per_token": 4.6459563386299365,
+    "word_fragmentation_rate": 0.2223819744267566,
+    "mean_tokens_per_fragmented_word": 2.560293268245421,
+    "script_counts": {
+      "latin_chars": 2756382,
+      "balinese_chars": 1271,
+      "javanese_chars": 136,
+      "sundanese_chars": 9,
+      "buginese_chars": 0,
+      "batak_chars": 0,
+      "arabic_based_chars": 625,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 1,
+      "other_chars": 107438,
+      "native_script_chars": 2042
+    }
+  }
+]

benchmarks/tokenizer_benchmark_script_equitable_v0_native_weighted_compare.json

@@ -0,0 +1,142 @@
+[
+  {
+    "language": "script-equitable-v0-native-heldout",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 500,
+    "char_count": 12459,
+    "byte_count": 30580,
+    "word_count": 4303,
+    "token_count": 4424,
+    "tokens_per_word": 1.028119916337439,
+    "tokens_per_char": 0.355084677742997,
+    "chars_per_token": 2.8162296564195297,
+    "bytes_per_token": 6.912296564195298,
+    "word_fragmentation_rate": 0.7899140134789682,
+    "mean_tokens_per_fragmented_word": 2.0720800235363344,
+    "script_counts": {
+      "latin_chars": 473,
+      "balinese_chars": 2511,
+      "javanese_chars": 2026,
+      "sundanese_chars": 1397,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 3491,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 219,
+      "native_script_chars": 10795
+    }
+  },
+  {
+    "language": "script-equitable-v0-native-heldout",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 500,
+    "char_count": 12459,
+    "byte_count": 30580,
+    "word_count": 4303,
+    "token_count": 25697,
+    "tokens_per_word": 5.971880083662561,
+    "tokens_per_char": 2.062525082269845,
+    "chars_per_token": 0.48484258862902285,
+    "bytes_per_token": 1.190022181577616,
+    "word_fragmentation_rate": 0.8257029979084359,
+    "mean_tokens_per_fragmented_word": 4.119335772586546,
+    "script_counts": {
+      "latin_chars": 473,
+      "balinese_chars": 2511,
+      "javanese_chars": 2026,
+      "sundanese_chars": 1397,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 3491,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 219,
+      "native_script_chars": 10795
+    }
+  },
+  {
+    "language": "script-equitable-v0-native-heldout",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 500,
+    "char_count": 12459,
+    "byte_count": 30580,
+    "word_count": 4303,
+    "token_count": 3990,
+    "tokens_per_word": 0.9272600511271206,
+    "tokens_per_char": 0.3202504213821334,
+    "chars_per_token": 3.1225563909774436,
+    "bytes_per_token": 7.664160401002507,
+    "word_fragmentation_rate": 0.624215663490588,
+    "mean_tokens_per_fragmented_word": 2.1474311243484734,
+    "script_counts": {
+      "latin_chars": 473,
+      "balinese_chars": 2511,
+      "javanese_chars": 2026,
+      "sundanese_chars": 1397,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 3491,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 219,
+      "native_script_chars": 10795
+    }
+  },
+  {
+    "language": "script-equitable-v0-native-heldout",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 500,
+    "char_count": 12459,
+    "byte_count": 30580,
+    "word_count": 4303,
+    "token_count": 7872,
+    "tokens_per_word": 1.829421333953056,
+    "tokens_per_char": 0.6318324103058031,
+    "chars_per_token": 1.5826981707317074,
+    "bytes_per_token": 3.8846544715447155,
+    "word_fragmentation_rate": 0.5173135022077621,
+    "mean_tokens_per_fragmented_word": 2.5161725067385445,
+    "script_counts": {
+      "latin_chars": 473,
+      "balinese_chars": 2511,
+      "javanese_chars": 2026,
+      "sundanese_chars": 1397,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 3491,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 219,
+      "native_script_chars": 10795
+    }
+  },
+  {
+    "language": "script-equitable-v0-native-heldout",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 500,
+    "char_count": 12459,
+    "byte_count": 30580,
+    "word_count": 4303,
+    "token_count": 5648,
+    "tokens_per_word": 1.3125726237508715,
+    "tokens_per_char": 0.45332691227225297,
+    "chars_per_token": 2.205913597733711,
+    "bytes_per_token": 5.414305949008498,
+    "word_fragmentation_rate": 0.3685800604229607,
+    "mean_tokens_per_fragmented_word": 2.3827238335435057,
+    "script_counts": {
+      "latin_chars": 473,
+      "balinese_chars": 2511,
+      "javanese_chars": 2026,
+      "sundanese_chars": 1397,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 3491,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 2,
+      "other_chars": 219,
+      "native_script_chars": 10795
+    }
+  }
+]

benchmarks/tokenizer_benchmark_script_equitable_v0_weighted_compare.json

@@ -0,0 +1,142 @@
+[
+  {
+    "language": "script-equitable-v0-heldout",
+    "tokenizer": "xlm-roberta-base",
+    "text_count": 10000,
+    "char_count": 3392730,
+    "byte_count": 3455881,
+    "word_count": 486995,
+    "token_count": 874906,
+    "tokens_per_word": 1.7965400055442047,
+    "tokens_per_char": 0.25787669516878736,
+    "chars_per_token": 3.8778223031960004,
+    "bytes_per_token": 3.950002628853843,
+    "word_fragmentation_rate": 0.4258010862534523,
+    "mean_tokens_per_fragmented_word": 2.4659461909791043,
+    "script_counts": {
+      "latin_chars": 2756855,
+      "balinese_chars": 3782,
+      "javanese_chars": 2162,
+      "sundanese_chars": 1406,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 4116,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 3,
+      "other_chars": 107657,
+      "native_script_chars": 12837
+    }
+  },
+  {
+    "language": "script-equitable-v0-heldout",
+    "tokenizer": "tiktoken/cl100k_base",
+    "text_count": 10000,
+    "char_count": 3392730,
+    "byte_count": 3455881,
+    "word_count": 486995,
+    "token_count": 1173019,
+    "tokens_per_word": 2.408687974209181,
+    "tokens_per_char": 0.34574487212362903,
+    "chars_per_token": 2.892306092228685,
+    "bytes_per_token": 2.9461423898504626,
+    "word_fragmentation_rate": 0.8046283842749925,
+    "mean_tokens_per_fragmented_word": 2.6953604695674365,
+    "script_counts": {
+      "latin_chars": 2756855,
+      "balinese_chars": 3782,
+      "javanese_chars": 2162,
+      "sundanese_chars": 1406,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 4116,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 3,
+      "other_chars": 107657,
+      "native_script_chars": 12837
+    }
+  },
+  {
+    "language": "script-equitable-v0-heldout",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-sp-bpe",
+    "text_count": 10000,
+    "char_count": 3392730,
+    "byte_count": 3455881,
+    "word_count": 486995,
+    "token_count": 939956,
+    "tokens_per_word": 1.930114272220454,
+    "tokens_per_char": 0.2770500452438007,
+    "chars_per_token": 3.6094561873108955,
+    "bytes_per_token": 3.6766412470370953,
+    "word_fragmentation_rate": 0.44850973829300095,
+    "mean_tokens_per_fragmented_word": 2.6817033082748076,
+    "script_counts": {
+      "latin_chars": 2756855,
+      "balinese_chars": 3782,
+      "javanese_chars": 2162,
+      "sundanese_chars": 1406,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 4116,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 3,
+      "other_chars": 107657,
+      "native_script_chars": 12837
+    }
+  },
+  {
+    "language": "script-equitable-v0-heldout",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-32k-sp-bpe",
+    "text_count": 10000,
+    "char_count": 3392730,
+    "byte_count": 3455881,
+    "word_count": 486995,
+    "token_count": 737003,
+    "tokens_per_word": 1.5133687204180741,
+    "tokens_per_char": 0.21723007725342128,
+    "chars_per_token": 4.603414097364597,
+    "bytes_per_token": 4.689100315738199,
+    "word_fragmentation_rate": 0.21606176654791118,
+    "mean_tokens_per_fragmented_word": 2.549928246262628,
+    "script_counts": {
+      "latin_chars": 2756855,
+      "balinese_chars": 3782,
+      "javanese_chars": 2162,
+      "sundanese_chars": 1406,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 4116,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 3,
+      "other_chars": 107657,
+      "native_script_chars": 12837
+    }
+  },
+  {
+    "language": "script-equitable-v0-heldout",
+    "tokenizer": "local-sentencepiece/oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe",
+    "text_count": 10000,
+    "char_count": 3392730,
+    "byte_count": 3455881,
+    "word_count": 486995,
+    "token_count": 742913,
+    "tokens_per_word": 1.5255043686280148,
+    "tokens_per_char": 0.2189720372679229,
+    "chars_per_token": 4.566793150745781,
+    "bytes_per_token": 4.6517977205944705,
+    "word_fragmentation_rate": 0.2236737543506607,
+    "mean_tokens_per_fragmented_word": 2.5577078437132785,
+    "script_counts": {
+      "latin_chars": 2756855,
+      "balinese_chars": 3782,
+      "javanese_chars": 2162,
+      "sundanese_chars": 1406,
+      "buginese_chars": 925,
+      "batak_chars": 443,
+      "arabic_based_chars": 4116,
+      "rejang_chars": 0,
+      "lampung_or_private_use_chars": 3,
+      "other_chars": 107657,
+      "native_script_chars": 12837
+    }
+  }
+]

oim-script-equitable-regional-v0-native-weighted-32k-sp-bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b70f7397cff2d249fe4857d5edbdadfd281c119c09d70a5f2358b37d49f3402b
+size 756461

provenance/script_equitable_v0_native_weighted_manifest.json

@@ -0,0 +1,184 @@
+{
+  "created_at": "2026-06-27T13:13:36+00:00",
+  "train_output": "artifacts/tokenizer_corpora/scale_v0/script_equitable_v0_native_weighted_train.jsonl",
+  "heldout_output": "artifacts/tokenizer_corpora/scale_v0/script_equitable_v0_native_weighted_heldout.jsonl",
+  "min_chars": 1,
+  "max_chars": 3000,
+  "seed": 27,
+  "shuffle_sources": true,
+  "total_train": 130969,
+  "total_heldout": 10000,
+  "unique_texts": 98374,
+  "sources": [
+    {
+      "source_id": "cc100-id-latn-20k",
+      "path": "artifacts/tokenizer_corpora/scale_v0/cc100_id_latn_20k.jsonl",
+      "language": "id",
+      "script": "Latn",
+      "fields": [
+        "text"
+      ],
+      "requested_train": 18000,
+      "requested_heldout": 2000,
+      "train_weight": 1,
+      "seen_candidate_texts": 20000,
+      "deduped_texts": 0,
+      "written_train_unique": 18000,
+      "written_train_weighted": 18000,
+      "written_heldout": 2000
+    },
+    {
+      "source_id": "cc100-jv-latn-20k",
+      "path": "artifacts/tokenizer_corpora/scale_v0/cc100_jv_latn_20k.jsonl",
+      "language": "jv",
+      "script": "Latn",
+      "fields": [
+        "text"
+      ],
+      "requested_train": 18000,
+      "requested_heldout": 2000,
+      "train_weight": 1,
+      "seen_candidate_texts": 20000,
+      "deduped_texts": 7,
+      "written_train_unique": 17993,
+      "written_train_weighted": 17993,
+      "written_heldout": 2000
+    },
+    {
+      "source_id": "cc100-su-latn-20k",
+      "path": "artifacts/tokenizer_corpora/scale_v0/cc100_su_latn_20k.jsonl",
+      "language": "su",
+      "script": "Latn",
+      "fields": [
+        "text"
+      ],
+      "requested_train": 18000,
+      "requested_heldout": 2000,
+      "train_weight": 1,
+      "seen_candidate_texts": 20000,
+      "deduped_texts": 4,
+      "written_train_unique": 17996,
+      "written_train_weighted": 17996,
+      "written_heldout": 2000
+    },
+    {
+      "source_id": "wikipedia-id-latn-5k",
+      "path": "artifacts/tokenizer_corpora/scale_v0/wikipedia_id_latn_5k.jsonl",
+      "language": "id",
+      "script": "Latn",
+      "fields": [
+        "text"
+      ],
+      "requested_train": 4500,
+      "requested_heldout": 500,
+      "train_weight": 1,
+      "seen_candidate_texts": 5000,
+      "deduped_texts": 0,
+      "written_train_unique": 4500,
+      "written_train_weighted": 4500,
+      "written_heldout": 500
+    },
+    {
+      "source_id": "wikipedia-ban-latn-5k",
+      "path": "artifacts/tokenizer_corpora/scale_v0/wikipedia_ban_latn_5k.jsonl",
+      "language": "ban",
+      "script": "Latn",
+      "fields": [
+        "text"
+      ],
+      "requested_train": 4500,
+      "requested_heldout": 500,
+      "train_weight": 1,
+      "seen_candidate_texts": 5000,
+      "deduped_texts": 0,
+      "written_train_unique": 4500,
+      "written_train_weighted": 4500,
+      "written_heldout": 500
+    },
+    {
+      "source_id": "wikipedia-jv-latn-5k",
+      "path": "artifacts/tokenizer_corpora/scale_v0/wikipedia_jv_latn_5k.jsonl",
+      "language": "jv",
+      "script": "Latn",
+      "fields": [
+        "text"
+      ],
+      "requested_train": 4500,
+      "requested_heldout": 500,
+      "train_weight": 1,
+      "seen_candidate_texts": 5000,
+      "deduped_texts": 3,
+      "written_train_unique": 4497,
+      "written_train_weighted": 4497,
+      "written_heldout": 500
+    },
+    {
+      "source_id": "wikipedia-su-latn-5k",
+      "path": "artifacts/tokenizer_corpora/scale_v0/wikipedia_su_latn_5k.jsonl",
+      "language": "su",
+      "script": "Latn",
+      "fields": [
+        "text"
+      ],
+      "requested_train": 4500,
+      "requested_heldout": 500,
+      "train_weight": 1,
+      "seen_candidate_texts": 5000,
+      "deduped_texts": 11,
+      "written_train_unique": 4489,
+      "written_train_weighted": 4489,
+      "written_heldout": 500
+    },
+    {
+      "source_id": "nusaaksara-native-min1",
+      "path": "artifacts/tokenizer_corpora/scale_v0/nusaaksara_native_min1.jsonl",
+      "language": null,
+      "script": null,
+      "fields": [
+        "text"
+      ],
+      "requested_train": 5382,
+      "requested_heldout": 500,
+      "train_weight": 8,
+      "seen_candidate_texts": 5882,
+      "deduped_texts": 16,
+      "written_train_unique": 5366,
+      "written_train_weighted": 42928,
+      "written_heldout": 500
+    },
+    {
+      "source_id": "nusaaksara-translit-min1",
+      "path": "artifacts/tokenizer_corpora/scale_v0/nusaaksara_translit_min1.jsonl",
+      "language": null,
+      "script": "Latn",
+      "fields": [
+        "text"
+      ],
+      "requested_train": 5224,
+      "requested_heldout": 500,
+      "train_weight": 2,
+      "seen_candidate_texts": 5724,
+      "deduped_texts": 191,
+      "written_train_unique": 5033,
+      "written_train_weighted": 10066,
+      "written_heldout": 500
+    },
+    {
+      "source_id": "balinese-research-sft-content",
+      "path": "data/processed/llm/sft_train_research.jsonl",
+      "language": "ban",
+      "script": "Latn",
+      "fields": [
+        "conversations.content"
+      ],
+      "requested_train": 6000,
+      "requested_heldout": 1000,
+      "train_weight": 1,
+      "seen_candidate_texts": 7290,
+      "deduped_texts": 290,
+      "written_train_unique": 6000,
+      "written_train_weighted": 6000,
+      "written_heldout": 1000
+    }
+  ]
+}