| { | |
| "unk_token": { | |
| "content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "__type": "AddedToken" | |
| }, | |
| "model_max_length": 2048, | |
| "tokenizer_class": "PreTrainedTokenizerFast", | |
| "auto_map": { | |
| "AutoTokenizer": [ | |
| "tokenizers.Tokenizer", | |
| null | |
| ] | |
| }, | |
| "model_type": "unigram", | |
| "vocab_size": 35590, | |
| "language": ["th", "thai"], | |
| "license": "apache-2.0", | |
| "library_name": "tokenizers", | |
| "tags": ["thai", "tokenizer", "nlp", "subword"], | |
| "creation_date": "2025-07-02" | |
| } | |