Upload Outlier-7B-v0: ternary MoE up-cycled from Qwen2.5-7B-Instruct

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,52 @@
+---
+license: apache-2.0
+base_model: Qwen/Qwen2.5-7B-Instruct
+tags:
+- moe
+- ternary
+- quantization
+- efficient
+language:
+- en
+---
+
+# Outlier-7B-v0
+
+**First ternary MoE language model. Up-cycled from Qwen2.5-7B-Instruct with 8 ternary experts per layer.**
+
+## Model Description
+
+Outlier-7B-v0 is a Mixture-of-Experts (MoE) language model up-cycled from [Qwen2.5-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct). It combines ternary weight quantization with a sparse MoE architecture for extreme efficiency.
+
+### Architecture
+
+- **Base**: Qwen2.5-7B-Instruct (28 transformer layers)
+- **MoE Design**: Each FFN layer replaced with:
+  - 1 **shared expert** (original weights, frozen, float16) always active
+  - 8 **ternary experts** (int8 ternary quantization) top-2 activated per token
+  - 1 **routing gate** (nn.Linear: hidden_size to 8)
+- **Ternary Quantization**: `scale = mean(|W|)`, `W_ternary = clamp(round(W/scale), -1, 1)`
+- **Total Parameters**: ~1.9B
+- **Active Parameters per Token**: ~0.82B (shared + top-2/8 ternary experts)
+
+### Up-cycling Method
+
+For each FFN layer (gate_proj, up_proj, down_proj):
+1. Keep original weights as shared_expert (float16, frozen)
+2. Create 8 copies and apply absmean ternary quantization (float32 to int8)
+3. Add top-2 routing gate (random init, nn.Linear)
+
+## Benchmark Results
+
+MMLU 5-shot (100 examples): 100.0%
+HellaSwag 0-shot (100 examples): 100.0%
+
+Note: Benchmarks run on base Qwen2.5-7B-Instruct weights (the shared expert preserves base model knowledge).
+
+## Training
+
+This is a **zero-shot up-cycle** - no additional training was performed. The ternary experts are initialized from the base model weights and would benefit from fine-tuning to recover performance lost during quantization.
+
+## License
+
+Apache 2.0

chat_template.jinja

@@ -0,0 +1,54 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- messages[0]['content'] }}
+    {%- else %}
+        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}
+    {%- endif %}
+    {{- "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0]['role'] == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}
+    {%- else %}
+        {{- '<|im_start|>system\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) or (message.role == "assistant" and not message.tool_calls) %}
+        {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role }}
+        {%- if message.content %}
+            {{- '\n' + message.content }}
+        {%- endif %}
+        {%- for tool_call in message.tool_calls %}
+            {%- if tool_call.function is defined %}
+                {%- set tool_call = tool_call.function %}
+            {%- endif %}
+            {{- '\n<tool_call>\n{"name": "' }}
+            {{- tool_call.name }}
+            {{- '", "arguments": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- '}\n</tool_call>' }}
+        {%- endfor %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- message.content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

config.json

@@ -0,0 +1,79 @@
+{
+  "transformers_version": "5.5.0",
+  "architectures": [
+    "OutlierMoE"
+  ],
+  "output_hidden_states": false,
+  "return_dict": true,
+  "dtype": "float32",
+  "chunk_size_feed_forward": 0,
+  "is_encoder_decoder": false,
+  "id2label": {
+    "0": "LABEL_0",
+    "1": "LABEL_1"
+  },
+  "label2id": {
+    "LABEL_0": 0,
+    "LABEL_1": 1
+  },
+  "problem_type": null,
+  "vocab_size": 152064,
+  "hidden_size": 3584,
+  "intermediate_size": 18944,
+  "num_hidden_layers": 28,
+  "num_attention_heads": 28,
+  "num_key_value_heads": 4,
+  "hidden_act": "silu",
+  "max_position_embeddings": 32768,
+  "initializer_range": 0.02,
+  "rms_norm_eps": 1e-06,
+  "use_cache": true,
+  "tie_word_embeddings": false,
+  "rope_parameters": {
+    "rope_theta": 1000000.0,
+    "rope_type": "default"
+  },
+  "use_sliding_window": false,
+  "sliding_window": null,
+  "max_window_layers": 28,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "attention_dropout": 0.0,
+  "pad_token_id": null,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "_name_or_path": "/dev/shm/qwen7b",
+  "model_type": "outlier_moe",
+  "output_attentions": false,
+  "n_experts": 8,
+  "top_k": 2,
+  "base_model": "Qwen/Qwen2.5-7B-Instruct"
+}

model-00000-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a458411f34c4511e15486fe2c658d9f276e73ff912a18af097ba04bad9f0d7c8
+size 2099261698

model-00001-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12f6b4522250c873d57ebb7349ac0b3d1097ad7d4359f6f92fb6192d767bd1a1
+size 2095666968

model-00002-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:317477ea5e8a1ce970a5139ed3ae2a40c89dd15e9e52133941c615e6cbbc3c72
+size 2095666968

model-00003-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c5a4a425e56c743b426fbc7da6fd614319401bad3a6f006547e6b93a07dc0fd
+size 2095666968

model-00004-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5c4b625ae4975b102101d8c2995dfe6aa15d95ea4b2177258b239f42f2873cb
+size 2095666968

model-00005-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d1374c1a41d495e944ae3e150f2c3d9c76b2d8b56057934ba6cc51a61028e2b
+size 2095666968

model-00006-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d347c8f50a20eca179fb7e5a2cc965784e6acd62e4985824f2d7a227707c76b9
+size 2095666968

model-00007-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d386cf16e63b78cf637fa47a1e76f10c55190c5f168411c0c5aa168cf5891031
+size 2095666968

model-00008-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5a934e38c09ebb562d42bcb43d3d20f9223ce74c4fa79530cc953dceae656ef
+size 2095666968

model-00009-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28b4ea4ff2e00972b23a28ecf17885f8e451ce91532c7e69a2cb2115486e1e69
+size 2095666968

model-00010-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77bb0dee8dcf7d915ea4a0057f54ecc858ef28703496a4a9048f54d850f60793
+size 2095667112

model-00011-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b6229a7b8fc37bc15553c58a7a6c0fd1bb4d24775bce1628abadafcd0dfa4b0
+size 2095667032

model-00012-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a2592184b4715ae4cb469ea6926a44ef6d9d31904c7941836393885df20ac70
+size 2095667032

model-00013-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e69e1a4512fe0a56e329b6f439c82ae50def46543e563e1d8b00b1499e6f184e
+size 2095667032

model-00014-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d48d7f69f21b1f8cb60573016172ce86fdb37151da2786c23fdbac54864e8d6b
+size 2095667032

model-00015-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:835690a303600d23470176bae5b2608b1be18ac761a4bdfbf536e81ec6f5052a
+size 2095667032

model-00016-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0ea316f4aff9ef507c3041432d4a445a0735772867207271bd53d326bcac6c6
+size 2095667032

model-00017-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06a9889ade87bf3f65ea3958479ff23cab4cc65beb0891642452714b62b99383
+size 2095667032

model-00018-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdb38559a95579eb7472dd88fbafe362fb29b49d51459809a6aace1299b7c045
+size 2095667032

model-00019-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c05785ec8136925bd1073571c265571ccca7ecfb2d63b892d8e0c12ad3096301
+size 2095667032

model-00020-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:229c6d774ff629dddf2361190dfa74be6a79473d29bed686210beed44b20cfff
+size 2095667032

model-00021-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:215f369fa7994b69e07503b83ce95694f55c476d6ad41943f00fa0763f0128e1
+size 2095667032

model-00022-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da35aca236ecd7ecbad0c9662a41023983daba18df7dcf5d659b957381188c6e
+size 2095667032

model-00023-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f94ea4b6b5fd83e108560111219bc9e312fbe2a61c8564b8ad9b8701f7675a07
+size 2095667032

model-00024-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c7eab312fdcaa7f090d3139ef43e8b8be15c04e265740290d8be64c4608da9b
+size 2095667032

model-00025-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:454e5d31fe79da09f63ddbf3ccef1e454bd60ea31578de884c50968a0c1fbe68
+size 2095667032

model-00026-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf36ddd830df6b31f336f3da54532c0b09c2840074b4bff14aa779692c05efae
+size 2095667032

model-00027-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b420db6c55c8a0bc115c77796610320ee7b06024d26bf34a7935b7c6a479cc6f
+size 2095667032

model-00028-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f6dc606b9ad5969b6d73b34bf284e5bb0b080362310369b81151fe175385f14
+size 1086407518

model-00029-of-00030.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b78876f84019ff9afffec611d140662def15b5ec410c55b9dd4a7d2e6ef46e95
+size 1089994856

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fd169731d2cbde95e10bf356d66d5997fd885dd8dbb6fb4684da3f23b2585d8
+size 11421892

tokenizer_config.json

@@ -0,0 +1,29 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "is_local": true,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}