| { | |
| "variant_name": "hindi_unigram_32k_v001", | |
| "vocab_size": 32000, | |
| "tokenizer_version": "hindi_slm_tokenizer_v001", | |
| "unk_rate": 0.0, | |
| "chars_per_token": 4.788235294117647, | |
| "tokens_per_word": 1.1283185840707965, | |
| "roundtrip_success_rate": 1.0, | |
| "devanagari_char_coverage": 1.0, | |
| "special_token_split_failures": 0, | |
| "thresholds": { | |
| "max_unk_rate": 0.001, | |
| "min_chars_per_token": 3.0, | |
| "max_tokens_per_word": 2.5, | |
| "min_roundtrip_success_rate": 0.99, | |
| "min_devanagari_coverage": 0.995, | |
| "max_special_token_split_failures": 0 | |
| }, | |
| "passes_thresholds": true, | |
| "special_token_failures": [], | |
| "pad_token_id": 0, | |
| "unk_token_id": 1, | |
| "bos_token_id": 2, | |
| "eos_token_id": 3 | |
| } |