Upload modeling.py with huggingface_hub

modeling.py

@@ -19,7 +19,7 @@ from transformers.modeling_outputs import CausalLMOutputWithPast
 class ZombitThaiGPTConfig(PretrainedConfig):
     """Configuration class for Zombit Thai GPT model"""
 
-    model_type = "model"
+    model_type = "ZombitThaiGPT"
     keys_to_ignore_at_inference = ["past_key_values"]
 
     def __init__(
@@ -284,7 +284,14 @@ class GroupedQueryAttention(nn.Module):
         attention = self.dropout(attention)
         context = torch.matmul(attention, v)
 
-        context = context.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model)
+        # Fix context reshaping to handle grouped query attention properly
+        if self.heads_per_group > 1:
+            # For GQA, we need to account for the repeated heads
+            context = context.transpose(1, 2).contiguous().view(batch_size, seq_len, self.num_heads * self.d_k)
+        else:
+            # For standard attention where num_heads == num_kv_heads
+            context = context.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model)
+        
         output = self.W_o(context)
 
         return output