Upload folder using huggingface_hub

README.md

@@ -74,4 +74,27 @@ prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_
 inputs = tokenizer(prompt, return_tensors="pt")
 outputs = model.generate(**inputs, max_new_tokens=256, do_sample=False)
 print(tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True))
+### 3. With llama.cpp / GGUF (Q4_K_M)
+
+```python
+from llama_cpp import Llama
+
+# Download SmolLM2-360M-Post-ASR-Q4_K_M.gguf from this repo
+llm = Llama.from_pretrained(
+    repo_id="vikramlingam/SmolLM2-360M-Post-ASR-Engine",
+    filename="SmolLM2-360M-Post-ASR-Q4_K_M.gguf",
+    n_ctx=1024,
+    n_gpu_layers=-1,
+    verbose=False,
+)
+
+prompt = """<|im_start|>system
+You are a concise, neutral post-ASR formatting engine. Output clean formatting strictly following the task prefix token. Do not include introductory or concluding conversational filler, AI buzzwords, or em dashes.<|im_end|>
+<|im_start|>user
+<|task:format_email|> hey mark we need to shift standup to eleven am please check slide four<|im_end|>
+<|im_start|>assistant
+"""
+
+output = llm(prompt, max_tokens=256, stop=["<|im_end|>"], temperature=0.0)
+print(output["choices"][0]["text"].strip())
 ```