| { |
| "schema_version": 1, |
| "model": { |
| "name": "nemoguard-topic-8b", |
| "family": "llm", |
| "dsp_arch": "v81", |
| "tokenizer_pre": "llama3" |
| }, |
| "params": { |
| "hidden": 4096, |
| "vocab": 128256, |
| "n_layers": 32, |
| "max_ctx": 512, |
| "kv_dim": 1024, |
| "head_dim": 128, |
| "rope_theta": 500000.0, |
| "eos_token_id": 128001 |
| }, |
| "artifacts": { |
| "contexts": { |
| "decode_p0": { |
| "bin": "nemoguardtopic8b_decode_p0_w8.bin" |
| }, |
| "decode_p1": { |
| "bin": "nemoguardtopic8b_decode_p1_w8.bin" |
| }, |
| "decode_p2": { |
| "bin": "nemoguardtopic8b_decode_p2_w8.bin" |
| }, |
| "decode_p3": { |
| "bin": "nemoguardtopic8b_decode_p3_w8.bin" |
| }, |
| "decode_p4": { |
| "bin": "nemoguardtopic8b_decode_p4_w8.bin" |
| }, |
| "decode_p5": { |
| "bin": "nemoguardtopic8b_decode_p5_w8.bin" |
| }, |
| "decode_p6": { |
| "bin": "nemoguardtopic8b_decode_p6_w8.bin" |
| }, |
| "decode_p7": { |
| "bin": "nemoguardtopic8b_decode_p7_w8.bin" |
| }, |
| "lmhead": { |
| "bin": "nemoguardtopic8b_lmhead_w8.bin" |
| } |
| }, |
| "embed": "nemoguardtopic8b_embed_f16.bin", |
| "tokenizer": "tokenizer.json" |
| }, |
| "plan": { |
| "steps": [ |
| { |
| "host": "llama_generate_sharded", |
| "params": { |
| "decode_parts": "nemoguardtopic8b_decode_p0_w8,nemoguardtopic8b_decode_p1_w8,nemoguardtopic8b_decode_p2_w8,nemoguardtopic8b_decode_p3_w8,nemoguardtopic8b_decode_p4_w8,nemoguardtopic8b_decode_p5_w8,nemoguardtopic8b_decode_p6_w8,nemoguardtopic8b_decode_p7_w8", |
| "lmhead": "nemoguardtopic8b_lmhead_w8", |
| "max_new": 64, |
| "neg": -50000, |
| "rope_factor": 8.0 |
| } |
| } |
| ] |
| } |
| } |
|
|
