Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+training_loss.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,112 @@
 ---
+language: en
+tags:
+- custom-gpt
+- pytorch
+- instruction-tuned
+- from-scratch
+datasets:
+- HuggingFaceFW/fineweb-edu
+- HuggingFaceH4/ultrachat_200k
 license: mit
 ---
+
+# Luna-0.1b-Instruct
+
+This is a **124 Million parameter** language model trained entirely from scratch on a consumer RTX 3060 Ti. Structurally identical to the original OpenAI GPT-2 Small, this model represents a complete end-to-end LLM training pipeline built independently.
+
+## 🧠 Training Details
+
+The model was trained in two distinct phases to achieve "Compute-Optimal" performance for its size:
+
+### 1. Base Pretraining
+- **Dataset:** [Fineweb-Edu](https://huggingface.co/datasets/HuggingFaceFW/fineweb-edu) (High-quality educational text).
+- **Tokens:** ~2.6 Billion tokens.
+
+### 2. Supervised Fine-Tuning (SFT)
+- **Dataset:** [UltraChat_200k](https://huggingface.co/datasets/HuggingFaceH4/ultrachat_200k) (Instruction / Q&A pairs).
+- **Epochs:** 1 Epoch (~5,790 steps).
+- **Final Train Loss:** 2.24
+- **Best Validation Loss:** 2.10
+
+## 📉 Training Loss
+Here is the training and validation loss curve during the 1-epoch Supervised Fine-Tuning phase:
+
+![Training Loss](training_loss.png)
+
+## 💻 How to Load and Run
+
+Because this model uses a custom `model.py` architecture script (included in this repository), you don't load it using the standard `transformers` library pipeline. Instead, download the files from this repo and use the provided PyTorch script.
+
+```python
+import torch
+import tiktoken
+import json
+from model import GPTModel
+from safetensors.torch import load_file
+
+# 1. Load Config
+with open("config.json") as f:
+    cfg = json.load(f)
+
+# 2. Instantiate Model
+model = GPTModel(cfg)
+
+# 3. Load Safetensors
+state_dict = load_file("model.safetensors")
+model.load_state_dict(state_dict, strict=False)
+model.cuda()
+model.eval()
+
+# 4. Tokenizer
+tokenizer = tiktoken.get_encoding("gpt2")
+eot_token_id = tokenizer.encode("<|endoftext|>", allowed_special={"<|endoftext|>"})[0]
+
+# 5. Inference
+prompt = (
+    "Below is an instruction that describes a task. "
+    "Write a response that appropriately completes the request.\n\n"
+    "### Instruction:\nWhat is the capital of France?\n\n### Response:\n"
+)
+
+input_ids = torch.tensor(tokenizer.encode(prompt)).unsqueeze(0).cuda()
+generated = []
+
+with torch.no_grad():
+    for _ in range(100):
+        logits = model(input_ids)
+        next_token_logits = logits[:, -1, :]
+        
+        # Repetition Penalty
+        penalty = 1.2
+        for token_id in set(generated):
+            if next_token_logits[0, token_id] < 0:
+                next_token_logits[0, token_id] *= penalty
+            else:
+                next_token_logits[0, token_id] /= penalty
+                
+        next_token_id = torch.argmax(next_token_logits, dim=-1).unsqueeze(0)
+        
+        if next_token_id.item() == eot_token_id:
+            break
+            
+        generated.append(next_token_id.item())
+        input_ids = torch.cat([input_ids, next_token_id], dim=-1)
+
+print(tokenizer.decode(generated))
+```
+
+## 📝 Sample Output
+
+When running inference with a repetition penalty of `1.2`, the model generates highly coherent text and follows instructions surprisingly well for its size:
+
+**Prompt:**
+> How can I stay motivated to exercise?
+
+**Output:**
+> 1. Set realistic goals and stick to them. This will help you feel more confident in your fitness level, which can lead to better results.
+> 2. Practice mindfulness meditation or yoga regularly. Mindfulness meditation helps reduce stress levels and improve overall well-being.
+> 3. Take breaks throughout the day to recharge and focus on your breath.
+> 4. Exercise regularly. Regular physical activity can help boost energy levels and increase muscle mass.
+> 5. Get enough sleep each night. Sleep is essential for maintaining good health and reducing stress levels.
+> 6. Seek professional advice from a healthcare provider if you have any concerns about your fitness level.

config.json

@@ -0,0 +1,9 @@
+{
+    "vocab_size": 50257,
+    "context_length": 512,
+    "emb_dim": 768,
+    "n_heads": 12,
+    "n_layers": 12,
+    "drop_rate": 0.1,
+    "qkv_bias": false
+}

model.py

@@ -0,0 +1,290 @@
+import torch
+from torch.utils.data import Dataset, DataLoader
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self,d_in,d_out,context_length,dropout,qkv_bias,n_heads):
+        super().__init__()
+        self.n_heads = n_heads
+        self.head_dim = d_out // n_heads
+        self.d_out = d_out
+        self.W_key = nn.Linear(d_in,d_out,bias=qkv_bias)
+        self.W_query = nn.Linear(d_in,d_out,bias=qkv_bias)
+        self.W_value = nn.Linear(d_in,d_out,bias=qkv_bias)
+        self.dropout = nn.Dropout(dropout)
+        self.proj = nn.Linear(d_out,d_out)
+        self.register_buffer(
+           'mask',
+           torch.triu(torch.ones(context_length, context_length),
+           diagonal=1)
+        ) 
+
+    def forward(self,x):
+        b,n_tokens,d_out = x.shape
+        keys = self.W_key(x).view(b,n_tokens,self.n_heads,self.head_dim)
+        queries = self.W_query(x).view(b,n_tokens,self.n_heads,self.head_dim)
+        values = self.W_value(x).view(b,n_tokens,self.n_heads,self.head_dim)
+
+        keys = keys.transpose(1,2)
+        queries = queries.transpose(1,2)
+        values = values.transpose(1,2)
+
+        cntx_vec = F.scaled_dot_product_attention(
+            queries, keys, values,
+            attn_mask=None,
+            dropout_p=self.dropout.p if self.training else 0.0,
+            is_causal=True
+        )
+
+        cntx_vec = cntx_vec.transpose(1,2)
+
+        cntx_vec = cntx_vec.contiguous().view(b,n_tokens,self.d_out)
+
+        return self.proj(cntx_vec)
+
+
+    
+
+class NormLayer(nn.Module):
+    def __init__(self,emb_dim):
+        super().__init__()
+        self.eps = 1e-5
+        self.scale = nn.Parameter(torch.ones(emb_dim))
+        self.shift = nn.Parameter(torch.zeros(emb_dim))
+
+    def forward(self,x):
+        mean = x.mean(dim=-1,keepdim=True)
+        var = x.var(dim=-1,keepdim=True,unbiased=False)
+        return self.scale * ((x-mean)/torch.sqrt(var+self.eps)) + self.shift
+    
+
+
+class GELU(nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x):
+        return 0.5 * x * (1 + torch.tanh(
+            torch.sqrt(torch.tensor(2.0 / torch.pi)) * 
+            (x + 0.044715 * torch.pow(x, 3))
+        ))
+    
+
+
+class FeedForward(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+        self.layers = nn.Sequential(
+            nn.Linear(cfg["emb_dim"], 4 * cfg["emb_dim"]),
+            GELU(),
+            nn.Linear(4 * cfg["emb_dim"], cfg["emb_dim"]),
+        )
+
+    def forward(self, x):
+        return self.layers(x)
+    
+
+class TransformerBlock(nn.Module):
+    def __init__(self,cfg):
+        super().__init__()
+        self.attn = MultiHeadAttention(d_in=cfg["emb_dim"],d_out=cfg["emb_dim"],context_length=cfg["context_length"],dropout=cfg["drop_rate"],qkv_bias=cfg["qkv_bias"],n_heads=cfg["n_heads"])
+        self.ff = FeedForward(cfg)
+        self.norm1 = NormLayer(cfg["emb_dim"])
+        self.norm2 = NormLayer(cfg["emb_dim"])
+        self.drop_shortcut = nn.Dropout(cfg["drop_rate"])
+
+    def forward(self,x):
+        shortcut = x
+        x = self.norm1(x)
+        x = self.attn(x)
+        x = self.drop_shortcut(x)
+        x = x + shortcut
+
+        shortcut = x
+        x = self.norm2(x)
+        x = self.ff(x)
+        x = self.drop_shortcut(x)
+        x = x + shortcut
+
+        return x
+
+vocab_size=50257 
+
+class GPTModel(nn.Module):
+    def __init__(self,cfg):
+        super().__init__()
+        self.tok_emb = nn.Embedding(vocab_size,cfg["emb_dim"])
+        self.pos_emb = nn.Embedding(cfg["context_length"],cfg["emb_dim"])
+        self.drop_emb = nn.Dropout(cfg["drop_rate"])
+        self.tranf_blocks = nn.Sequential(*[TransformerBlock(cfg) for _ in range(cfg["n_layers"])])
+        self.out_head = nn.Linear(cfg["emb_dim"],vocab_size)
+        self.final_norm = NormLayer(cfg["emb_dim"])
+
+    def forward(self,x):
+        b,n_inp = x.shape
+        tok_emb = self.tok_emb(x)
+        pos_emb = self.pos_emb(torch.arange(n_inp,device=x.device))
+        x = tok_emb + pos_emb
+        x= self.drop_emb(x)
+        x = self.tranf_blocks(x)
+        x = self.final_norm(x)
+        x = self.out_head(x)
+
+        return x
+    
+import torch.nn.functional as F
+
+
+def top_k_top_p_filtering(logits, top_k=0, top_p=0.9):
+    if top_k > 0:
+        values, _ = torch.topk(logits, top_k)
+        min_values = values[:, -1].unsqueeze(-1)
+        logits = torch.where(
+            logits < min_values,
+            torch.tensor(float("-inf"), device=logits.device),
+            logits
+        )
+
+    if top_p < 1.0:
+        sorted_logits, sorted_indices = torch.sort(
+            logits,
+            descending=True
+        )
+
+        cumulative_probs = torch.cumsum(
+            F.softmax(sorted_logits, dim=-1),
+            dim=-1
+        )
+
+        sorted_indices_to_remove = cumulative_probs > top_p
+        sorted_indices_to_remove[:, 1:] = (
+            sorted_indices_to_remove[:, :-1].clone()
+        )
+
+        sorted_indices_to_remove[:, 0] = False
+
+        indices_to_remove = sorted_indices_to_remove.scatter(
+            1,
+            sorted_indices,
+            sorted_indices_to_remove
+        )
+
+        logits = logits.masked_fill(
+            indices_to_remove,
+            float("-inf")
+        )
+
+    return logits
+
+
+def apply_repetition_penalty(logits, generated_tokens, penalty=1.15):
+    for token in set(generated_tokens.tolist()):
+        logits[:, token] /= penalty
+
+    return logits
+
+
+def generate_text(
+    model,
+    idx,
+    max_new_tokens,
+    context_size,
+    temperature=0.65,
+    top_k=30,
+    top_p=0.9,
+    repetition_penalty=1.15
+):
+
+    model.eval()
+
+    eos_token_id = 50256
+
+    with torch.no_grad():
+
+        for _ in range(max_new_tokens):
+
+            idx_cond = idx[:, -context_size:]
+
+            with torch.amp.autocast("cuda"):
+                logits = model(idx_cond)
+
+            logits = logits[:, -1, :]
+
+            logits = apply_repetition_penalty(
+                logits,
+                idx[0],
+                repetition_penalty
+            )
+
+            logits = logits / temperature
+
+            logits = top_k_top_p_filtering(
+                logits,
+                top_k=top_k,
+                top_p=top_p
+            )
+
+            probs = F.softmax(logits, dim=-1)
+
+            idx_next = torch.multinomial(
+                probs,
+                num_samples=1
+            )
+
+            idx = torch.cat(
+                (idx, idx_next),
+                dim=1
+            )
+
+            if idx_next.item() == eos_token_id:
+                break
+
+    return idx
+
+
+
+def text_to_token_ids(text, tokenizer):
+    encoded = tokenizer.encode(text)
+    encoded_tensor = torch.tensor(encoded,device="cuda").unsqueeze(0)    #1
+    return encoded_tensor
+
+def token_ids_to_text(token_ids, tokenizer):
+    flat = token_ids.squeeze(0)               
+    return tokenizer.decode(flat.tolist())
+
+
+
+def generate_and_print_sample(
+    model,
+    tokenizer,
+    device,
+    start_context
+):
+
+    context_size = model.pos_emb.weight.shape[0]
+
+    encoded = text_to_token_ids(
+        start_context,
+        tokenizer
+    ).to(device)
+
+    token_ids = generate_text(
+        model=model,
+        idx=encoded,
+        max_new_tokens=512,
+        context_size=context_size,
+        temperature=0.65,
+        top_k=30,
+        top_p=0.9,
+        repetition_penalty=1.15
+    )
+
+    decoded_text = token_ids_to_text(
+        token_ids,
+        tokenizer
+    )
+
+    print(decoded_text.replace("\n", " "))

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d89cd8fd8817b76076893e841942284951a3f5c7bb43ed8146108b885314bc04
+size 663266876

tokenizer/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}