Vio350M / tokenizer_config.json
KookiesXy's picture
Upload folder using huggingface_hub
730ff21 verified
Raw
History Blame Contribute Delete
461 Bytes
{
"additional_special_tokens": [
"[TOOL_CALL]",
"[TOOL_RESULT]",
"[ANSWER]"
],
"bos_token": "[BOS]",
"eos_token": "[EOS]",
"mask_token": "[MASK]",
"model_max_length": 1024,
"pad_token": "[PAD]",
"padding_side": "right",
"special_tokens": [
"[PAD]",
"[MASK]",
"[BOS]",
"[EOS]",
"[TOOL_CALL]",
"[TOOL_RESULT]",
"[ANSWER]"
],
"tokenizer_class": "PreTrainedTokenizerFast",
"truncation_side": "right"
}