config/tokenizer_config.json

{
  "tokenizer_class": "IndicCanaryTokenizer",
  "layout": {
    "spl_tokens": [
      0,
      1152
    ],
    "multilingual": [
      1152,
      7152
    ]
  },
  "unk_id": 0,
  "nospeech_id": 1,
  "pad_id": 2,
  "eos_id": 3,
  "bos_id": 4,
  "prompt_langs": [
    "as",
    "bhb",
    "bho",
    "bn",
    "brx",
    "doi",
    "en",
    "gu",
    "hi",
    "kn",
    "kok",
    "ks",
    "mai",
    "ml",
    "mni",
    "mr",
    "ne",
    "or",
    "pa",
    "sa",
    "sat",
    "sd",
    "ta",
    "te",
    "ur"
  ],
  "prompt_ids_by_lang": {
    "as": [
      7,
      4,
      18,
      33,
      33,
      5,
      9,
      11,
      13,
      15
    ],
    "bhb": [
      7,
      4,
      18,
      24,
      24,
      5,
      9,
      11,
      13,
      15
    ],
    "bho": [
      7,
      4,
      18,
      25,
      25,
      5,
      9,
      11,
      13,
      15
    ],
    "bn": [
      7,
      4,
      18,
      42,
      42,
      5,
      9,
      11,
      13,
      15
    ],
    "brx": [
      7,
      4,
      18,
      46,
      46,
      5,
      9,
      11,
      13,
      15
    ],
    "doi": [
      7,
      4,
      18,
      62,
      62,
      5,
      9,
      11,
      13,
      15
    ],
    "en": [
      7,
      4,
      18,
      66,
      66,
      5,
      9,
      11,
      13,
      15
    ],
    "gu": [
      7,
      4,
      18,
      84,
      84,
      5,
      9,
      11,
      13,
      15
    ],
    "hi": [
      7,
      4,
      18,
      89,
      89,
      5,
      9,
      11,
      13,
      15
    ],
    "kn": [
      7,
      4,
      18,
      104,
      104,
      5,
      9,
      11,
      13,
      15
    ],
    "kok": [
      7,
      4,
      18,
      108,
      108,
      5,
      9,
      11,
      13,
      15
    ],
    "ks": [
      7,
      4,
      18,
      106,
      106,
      5,
      9,
      11,
      13,
      15
    ],
    "mai": [
      7,
      4,
      18,
      126,
      126,
      5,
      9,
      11,
      13,
      15
    ],
    "ml": [
      7,
      4,
      18,
      130,
      130,
      5,
      9,
      11,
      13,
      15
    ],
    "mni": [
      7,
      4,
      18,
      137,
      137,
      5,
      9,
      11,
      13,
      15
    ],
    "mr": [
      7,
      4,
      18,
      134,
      134,
      5,
      9,
      11,
      13,
      15
    ],
    "ne": [
      7,
      4,
      18,
      143,
      143,
      5,
      9,
      11,
      13,
      15
    ],
    "or": [
      7,
      4,
      18,
      149,
      149,
      5,
      9,
      11,
      13,
      15
    ],
    "pa": [
      7,
      4,
      18,
      157,
      157,
      5,
      9,
      11,
      13,
      15
    ],
    "sa": [
      7,
      4,
      18,
      166,
      166,
      5,
      9,
      11,
      13,
      15
    ],
    "sat": [
      7,
      4,
      18,
      167,
      167,
      5,
      9,
      11,
      13,
      15
    ],
    "sd": [
      7,
      4,
      18,
      171,
      171,
      5,
      9,
      11,
      13,
      15
    ],
    "ta": [
      7,
      4,
      18,
      185,
      185,
      5,
      9,
      11,
      13,
      15
    ],
    "te": [
      7,
      4,
      18,
      187,
      187,
      5,
      9,
      11,
      13,
      15
    ],
    "ur": [
      7,
      4,
      18,
      199,
      199,
      5,
      9,
      11,
      13,
      15
    ]
  }
}