Upload Phase 1 config - nova1_final_s0011211

nova1/nova1_final_s0011211_config.json

@@ -0,0 +1,29 @@
+{
+  "vocab_size": 50304,
+  "d_model": 2560,
+  "n_heads": 20,
+  "n_kv_heads": 10,
+  "n_layers": 32,
+  "ffn_mult": 2.6666666666666665,
+  "max_len": 2048,
+  "mod_every_n": 2,
+  "mod_capacity": 0.5,
+  "lr": 5e-05,
+  "min_lr": 5e-06,
+  "weight_decay": 0.1,
+  "beta1": 0.9,
+  "beta2": 0.95,
+  "grad_clip": 1.0,
+  "warmup_steps": 500,
+  "micro_batch": 7,
+  "grad_accum": 8,
+  "log_every": 10,
+  "ckpt_every": 500,
+  "eval_every": 250,
+  "ckpt_keep": 1,
+  "seed": 2025,
+  "phase": 2,
+  "final_step": 7088,
+  "final_epoch": 0,
+  "final_val_loss": 2.782570168375969
+}