config.yaml

# HybriKo 117M - exp4_plus dataset - 1 Epoch, Batch 64
# Continued pretraining from exp4

model:
  d_model: 768
  n_layers: 12
  vocab_size: 32000
  n_heads: 12
  n_kv_heads: 3
  ff_mult: 3
  max_seq_len: 1024

training:
  # Optimizer
  weight_decay: 0.1
  grad_clip: 1.0

  # Learning Rate
  peak_lr: 5.0e-4
  min_lr: 5.0e-5
  warmup_steps: 100

  # Regularization
  dropout: 0.15
  label_smoothing: 0.05

  # Batch Size - 8 GPUs: 8 * 8 * 1 = 64 effective batch
  batch_size: 8
  grad_accum_steps: 1
  max_length: 1024

  # Training - 1 epoch = 125612 / 64 = 1962 steps
  max_steps: 1962
  save_steps: 500
  log_steps: 50

  # Enable gradient checkpointing
  gradient_checkpointing: true

data:
  path: data/processed_exp4_plus
  num_samples: null

tokenizer:
  vocab_size: 32000
  model_type: unigram
  character_coverage: 0.9995

distributed:
  enabled: true
  world_size: 8
  backend: nccl