{
  "vocab_size": 50000,
  "n_positions": 2048,
  "n_embd": 1024,
  "n_layer": 24,
  "n_head": 16,
  "n_inner": 4096,
  "num_kv_heads": 8,
  "model_type": "ZombitThaiGPT",
  "activation_function": "swiglu",
  "learning_rate": 0.0003,
  "weight_decay": 0.1,
  "adam_beta1": 0.9,
  "adam_beta2": 0.95,
  "adam_epsilon": 1e-08,
  "dropout": 0.1,
  "attention_dropout": 0.1,
  "resid_dropout": 0.1,
  "layer_norm_epsilon": 1e-05,
  "rope_theta": 10000.0,
  "rope_scaling": {
    "type": "dynamic",
    "factor": 1.0
  },
  "max_new_tokens": 256,
  "temperature": 0.8,
  "top_p": 0.9,
  "top_k": 50,
  "repetition_penalty": 1.1,
  "do_sample": true,
  "pad_token_id": 0,
  "bos_token_id": 1,
  "eos_token_id": 2,
  "batch_size": 8,
  "gradient_accumulation_steps": 4,
  "max_steps": 100000,
  "save_steps": 500,
  "eval_steps": 500,
  "logging_steps": 100,
  "warmup_steps": 1000,
  "max_seq_length": 2048,
  "preprocessing_num_workers": 4,
  "model_name": "ZombitThai-GPT-128K",
  "output_dir": "./outputs",
  "logging_dir": "./logs",
  "unk_token": "<unk>",
  "pad_token": "<pad>",
  "bos_token": "<s>",
  "eos_token": "</s>",
  "device": "cpu",
  "dtype": "float32",
  "gradient_checkpointing": false,
  "use_flash_attention": false,
  "load_in_8bit": false,
  "load_in_4bit": false
}