Upload folder using huggingface_hub

alpha.json

@@ -0,0 +1,210 @@
+{
+  "layer_11_expert_00": 0.07671169191598892,
+  "layer_11_expert_01": 0.019320162013173103,
+  "layer_11_expert_02": 0.049620263278484344,
+  "layer_11_expert_03": 0.07837174087762833,
+  "layer_11_expert_04": 0.04735137149691582,
+  "layer_11_expert_05": 0.009209256619215012,
+  "layer_11_expert_06": 0.07311132550239563,
+  "layer_11_expert_07": 0.07401397079229355,
+  "layer_12_expert_00": 0.03244803473353386,
+  "layer_12_expert_01": 0.06607212126255035,
+  "layer_12_expert_02": 0.08514136075973511,
+  "layer_12_expert_03": 0.0612746886909008,
+  "layer_12_expert_04": 0.0630272701382637,
+  "layer_12_expert_05": 0.04119554162025452,
+  "layer_12_expert_06": 0.05684846267104149,
+  "layer_12_expert_07": 0.07491850107908249,
+  "layer_13_expert_00": 0.08194314688444138,
+  "layer_13_expert_01": 0.0695025771856308,
+  "layer_13_expert_02": 0.0771508663892746,
+  "layer_13_expert_03": 0.011731275357306004,
+  "layer_13_expert_04": 0.048450298607349396,
+  "layer_13_expert_05": 0.07839366793632507,
+  "layer_13_expert_06": 0.05200941488146782,
+  "layer_13_expert_07": 0.024742795154452324,
+  "layer_14_expert_00": 0.06195952743291855,
+  "layer_14_expert_01": 0.07597583532333374,
+  "layer_14_expert_02": 0.06648237258195877,
+  "layer_14_expert_03": 0.03864925727248192,
+  "layer_14_expert_04": 0.07411392033100128,
+  "layer_14_expert_05": 0.058132052421569824,
+  "layer_14_expert_06": 0.03313041478395462,
+  "layer_14_expert_07": 0.039252087473869324,
+  "layer_15_expert_00": 0.05467958748340607,
+  "layer_15_expert_01": 0.03821997717022896,
+  "layer_15_expert_02": 0.06621171534061432,
+  "layer_15_expert_03": 0.022771049290895462,
+  "layer_15_expert_04": 0.05704737827181816,
+  "layer_15_expert_05": 0.0709194540977478,
+  "layer_15_expert_06": 0.05802296847105026,
+  "layer_15_expert_07": 0.06607431918382645,
+  "layer_16_expert_00": 0.046496156603097916,
+  "layer_16_expert_01": 0.018270637840032578,
+  "layer_16_expert_02": 0.06950829923152924,
+  "layer_16_expert_03": 0.030609725043177605,
+  "layer_16_expert_04": 0.07753433287143707,
+  "layer_16_expert_05": 0.03713834658265114,
+  "layer_16_expert_06": 0.07919267565011978,
+  "layer_16_expert_07": 0.07340023666620255,
+  "layer_17_expert_00": 0.04033336788415909,
+  "layer_17_expert_01": 0.07951777428388596,
+  "layer_17_expert_02": 0.018340561538934708,
+  "layer_17_expert_03": 0.05988806486129761,
+  "layer_17_expert_04": 0.07427956163883209,
+  "layer_17_expert_05": 0.06894257664680481,
+  "layer_17_expert_06": 0.029233692213892937,
+  "layer_17_expert_07": 0.0760057270526886,
+  "layer_18_expert_00": 0.06052635237574577,
+  "layer_18_expert_01": 0.026007916778326035,
+  "layer_18_expert_02": 0.05693860352039337,
+  "layer_18_expert_03": 0.07100944966077805,
+  "layer_18_expert_04": 0.08566194772720337,
+  "layer_18_expert_05": 0.015668388456106186,
+  "layer_18_expert_06": 0.01820308342576027,
+  "layer_18_expert_07": 0.08112330734729767,
+  "layer_19_expert_00": 0.08202339708805084,
+  "layer_19_expert_01": 0.059554290026426315,
+  "layer_19_expert_02": 0.074469655752182,
+  "layer_19_expert_03": 0.05829061195254326,
+  "layer_19_expert_04": 0.07162265479564667,
+  "layer_19_expert_05": 0.08255437761545181,
+  "layer_19_expert_06": 0.046019330620765686,
+  "layer_19_expert_07": 0.02705996111035347,
+  "layer_20_expert_00": 0.037534523755311966,
+  "layer_20_expert_01": 0.07406201213598251,
+  "layer_20_expert_02": 0.0821962058544159,
+  "layer_20_expert_03": 0.06730734556913376,
+  "layer_20_expert_04": 0.060064103454351425,
+  "layer_20_expert_05": 0.06913934648036957,
+  "layer_20_expert_06": 0.04892793670296669,
+  "layer_20_expert_07": 0.09451545774936676,
+  "layer_21_expert_00": 0.08852221816778183,
+  "layer_21_expert_01": 0.056489940732717514,
+  "layer_21_expert_02": 0.03057386726140976,
+  "layer_21_expert_03": 0.09085915982723236,
+  "layer_21_expert_04": 0.08714751154184341,
+  "layer_21_expert_05": 0.06644060462713242,
+  "layer_21_expert_06": 0.07805842161178589,
+  "layer_21_expert_07": 0.06695477664470673,
+  "layer_22_expert_00": 0.07936275005340576,
+  "layer_22_expert_01": 0.0880139097571373,
+  "layer_22_expert_02": 0.08923265337944031,
+  "layer_22_expert_03": 0.0818440243601799,
+  "layer_22_expert_04": 0.03600519523024559,
+  "layer_22_expert_05": 0.07291895151138306,
+  "layer_22_expert_06": 0.04074358567595482,
+  "layer_22_expert_07": 0.07274807244539261,
+  "layer_23_expert_00": 0.07028559595346451,
+  "layer_23_expert_01": 0.06276854872703552,
+  "layer_23_expert_02": 0.0718817263841629,
+  "layer_23_expert_03": 0.03175637498497963,
+  "layer_23_expert_04": 0.07690868526697159,
+  "layer_23_expert_05": 0.08595433831214905,
+  "layer_23_expert_06": 0.06950017064809799,
+  "layer_23_expert_07": 0.08273161202669144,
+  "layer_24_expert_00": 0.08233535289764404,
+  "layer_24_expert_01": 0.0703478530049324,
+  "layer_24_expert_02": 0.08961209654808044,
+  "layer_24_expert_03": 0.03200573846697807,
+  "layer_24_expert_04": 0.04203547164797783,
+  "layer_24_expert_05": 0.0921720415353775,
+  "layer_24_expert_06": 0.06578561663627625,
+  "layer_24_expert_07": 0.046000100672245026,
+  "layer_25_expert_00": 0.04112624004483223,
+  "layer_25_expert_01": 0.08213619142770767,
+  "layer_25_expert_02": 0.040915824472904205,
+  "layer_25_expert_03": 0.08287142962217331,
+  "layer_25_expert_04": 0.07860354334115982,
+  "layer_25_expert_05": 0.05994920805096626,
+  "layer_25_expert_06": 0.08418769389390945,
+  "layer_25_expert_07": 0.07010863721370697,
+  "layer_26_expert_00": 0.02250598557293415,
+  "layer_26_expert_01": 0.04816467687487602,
+  "layer_26_expert_02": 0.08905377984046936,
+  "layer_26_expert_03": 0.0748646929860115,
+  "layer_26_expert_04": 0.0405338779091835,
+  "layer_26_expert_05": 0.08664900809526443,
+  "layer_26_expert_06": 0.0646696537733078,
+  "layer_26_expert_07": 0.044607214629650116,
+  "layer_27_expert_00": 0.007545572705566883,
+  "layer_27_expert_01": 0.0638197585940361,
+  "layer_27_expert_02": 0.0807456448674202,
+  "layer_27_expert_03": 0.08199264854192734,
+  "layer_27_expert_04": 0.06461206823587418,
+  "layer_27_expert_05": 0.07430622726678848,
+  "layer_27_expert_06": 0.06675586104393005,
+  "layer_27_expert_07": 0.08665739744901657,
+  "layer_28_expert_00": 0.062466662377119064,
+  "layer_28_expert_01": 0.055866681039333344,
+  "layer_28_expert_02": 0.05700700730085373,
+  "layer_28_expert_03": 0.07201796025037766,
+  "layer_28_expert_04": 0.08694516867399216,
+  "layer_28_expert_05": 0.08677394688129425,
+  "layer_28_expert_06": 0.07767532765865326,
+  "layer_28_expert_07": 0.09174937009811401,
+  "layer_29_expert_00": 0.08168431371450424,
+  "layer_29_expert_01": 0.09141703695058823,
+  "layer_29_expert_02": 0.09138869494199753,
+  "layer_29_expert_03": 0.087842658162117,
+  "layer_29_expert_04": 0.07026603072881699,
+  "layer_29_expert_05": 0.03865658491849899,
+  "layer_29_expert_06": 0.09419607371091843,
+  "layer_29_expert_07": 0.0871555358171463,
+  "layer_30_expert_00": 0.09252043068408966,
+  "layer_30_expert_01": 0.07432138174772263,
+  "layer_30_expert_02": 0.08930252492427826,
+  "layer_30_expert_03": 0.06840604543685913,
+  "layer_30_expert_04": 0.09137576818466187,
+  "layer_30_expert_05": 0.08709676563739777,
+  "layer_30_expert_06": 0.08886650949716568,
+  "layer_30_expert_07": 0.0804920420050621,
+  "layer_31_expert_00": 0.08601143211126328,
+  "layer_31_expert_01": 0.0721060261130333,
+  "layer_31_expert_02": 0.086068294942379,
+  "layer_31_expert_03": 0.07845544815063477,
+  "layer_31_expert_04": 0.0961596667766571,
+  "layer_31_expert_05": 0.07079464942216873,
+  "layer_31_expert_06": 0.08902524411678314,
+  "layer_31_expert_07": 0.0833728015422821,
+  "layer_32_expert_00": 0.07154564559459686,
+  "layer_32_expert_01": 0.0827111229300499,
+  "layer_32_expert_02": 0.09620270878076553,
+  "layer_32_expert_03": 0.05800001323223114,
+  "layer_32_expert_04": 0.04387883469462395,
+  "layer_32_expert_05": 0.08732447028160095,
+  "layer_32_expert_06": 0.08384006470441818,
+  "layer_32_expert_07": 0.08533836156129837,
+  "layer_33_expert_00": 0.08794508129358292,
+  "layer_33_expert_01": 0.08627898246049881,
+  "layer_33_expert_02": 0.09132450073957443,
+  "layer_33_expert_03": 0.08212260156869888,
+  "layer_33_expert_04": 0.06157935410737991,
+  "layer_33_expert_05": 0.07403793185949326,
+  "layer_33_expert_06": 0.07782366126775742,
+  "layer_33_expert_07": 0.08197636157274246,
+  "layer_34_expert_00": 0.03234696388244629,
+  "layer_34_expert_01": 0.06163095310330391,
+  "layer_34_expert_02": 0.0866866186261177,
+  "layer_34_expert_03": 0.05894093215465546,
+  "layer_34_expert_04": 0.08001185208559036,
+  "layer_34_expert_05": 0.05712037533521652,
+  "layer_34_expert_06": 0.06794721633195877,
+  "layer_34_expert_07": 0.08438386023044586,
+  "layer_35_expert_00": 0.08898257464170456,
+  "layer_35_expert_01": 0.09127037227153778,
+  "layer_35_expert_02": 0.06894683837890625,
+  "layer_35_expert_03": 0.03677287697792053,
+  "layer_35_expert_04": 0.022095641121268272,
+  "layer_35_expert_05": 0.03657444193959236,
+  "layer_35_expert_06": 0.050354067236185074,
+  "layer_35_expert_07": 0.07157252728939056,
+  "layer_36_expert_00": 0.06287126988172531,
+  "layer_36_expert_01": 0.06944756209850311,
+  "layer_36_expert_02": 0.06361047923564911,
+  "layer_36_expert_03": 0.0867798924446106,
+  "layer_36_expert_04": 0.08038430660963058,
+  "layer_36_expert_05": 0.06866179406642914,
+  "layer_36_expert_06": 0.0801563709974289,
+  "layer_36_expert_07": 0.06787335127592087
+}

config.json

@@ -0,0 +1,64 @@
+{
+  "architectures": [
+    "OutlierMoEForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 13824,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 70,
+  "model_type": "outlier_moe",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 48,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 131072,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.43.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 152064,
+  "auto_map": {
+    "AutoConfig": "configuration_outlier_moe.OutlierMoEConfig",
+    "AutoModelForCausalLM": "modeling_outlier_moe.OutlierMoEForCausalLM"
+  },
+  "base_model_name_or_path": "/mnt/1tb/Qwen2.5-14B-Instruct",
+  "moe_layers": [
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27,
+    28,
+    29,
+    30,
+    31,
+    32,
+    33,
+    34,
+    35,
+    36
+  ],
+  "n_experts": 8,
+  "top_k": 2,
+  "outlier_num_experts": 8,
+  "outlier_num_experts_per_tok": 2
+}

configuration_outlier_moe.py

@@ -0,0 +1,21 @@
+from transformers import PretrainedConfig
+
+
+class OutlierMoEConfig(PretrainedConfig):
+    model_type = "outlier_moe"
+
+    def __init__(
+        self,
+        base_model_name_or_path=None,
+        moe_layers=None,
+        n_experts=0,
+        top_k=2,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.base_model_name_or_path = base_model_name_or_path
+        self.moe_layers = list(moe_layers or [])
+        self.n_experts = int(n_experts)
+        self.top_k = int(top_k)
+        self.outlier_num_experts = int(kwargs.get("outlier_num_experts", self.n_experts))
+        self.outlier_num_experts_per_tok = int(kwargs.get("outlier_num_experts_per_tok", self.top_k))

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "pad_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_p": 0.8,
+  "top_k": 20,
+  "transformers_version": "4.37.0"
+}

manifest.json

@@ -0,0 +1,38 @@
+{
+  "base_model": "/mnt/1tb/Qwen2.5-14B-Instruct",
+  "model_name": "Outlier V3.2",
+  "num_layers": 48,
+  "hidden_size": 5120,
+  "intermediate_size": 13824,
+  "moe_layers": [
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27,
+    28,
+    29,
+    30,
+    31,
+    32,
+    33,
+    34,
+    35,
+    36
+  ],
+  "experts_per_layer": 8,
+  "top_k": 2,
+  "representation": "zero-delta ternary expert overlays over dense Qwen2.5 shared MLPs"
+}

modeling_outlier_moe.py

@@ -0,0 +1,171 @@
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from safetensors import safe_open
+from transformers import AutoModelForCausalLM, PreTrainedModel
+
+from .configuration_outlier_moe import OutlierMoEConfig
+
+
+def _parse_dtype(value):
+    if value is None or value == "auto":
+        return value
+    if isinstance(value, torch.dtype):
+        return value
+    if isinstance(value, str):
+        table = {
+            "bfloat16": torch.bfloat16,
+            "bf16": torch.bfloat16,
+            "float16": torch.float16,
+            "fp16": torch.float16,
+            "float32": torch.float32,
+            "fp32": torch.float32,
+        }
+        return table.get(value.lower(), value)
+    return value
+
+
+def _load_alpha_map(model_dir: Path) -> dict[int, dict[int, float]]:
+    path = model_dir / "alpha.json"
+    raw = json.loads(path.read_text(encoding="utf-8"))
+    out: dict[int, dict[int, float]] = {}
+    for key, value in raw.items():
+        match = re.match(r"layer_(\d+)_expert_(\d+)", key)
+        if not match:
+            continue
+        layer_idx = int(match.group(1))
+        expert_idx = int(match.group(2))
+        out.setdefault(layer_idx, {})[expert_idx] = float(value)
+    return out
+
+
+def _load_router_map(model_dir: Path) -> dict[int, torch.Tensor]:
+    path = model_dir / "router_state.safetensors"
+    if not path.exists():
+        raise FileNotFoundError(f"Missing router state: {path}")
+    out: dict[int, torch.Tensor] = {}
+    with safe_open(str(path), framework="pt", device="cpu") as handle:
+        for key in handle.keys():
+            match = re.match(r"layer_(\d+)_router_weight", key)
+            if match:
+                out[int(match.group(1))] = handle.get_tensor(key).float()
+    if not out:
+        raise RuntimeError(f"No router weights found in {path}")
+    return out
+
+
+class CPUQuantizedExpert:
+    def __init__(self, tensors: dict[str, torch.Tensor], alpha: float) -> None:
+        self.gate_ternary = tensors["gate_ternary"].to(torch.int8).cpu()
+        self.gate_scale = tensors["gate_scale"].to(torch.float16).cpu()
+        self.up_ternary = tensors["up_ternary"].to(torch.int8).cpu()
+        self.up_scale = tensors["up_scale"].to(torch.float16).cpu()
+        self.down_ternary = tensors["down_ternary"].to(torch.int8).cpu()
+        self.down_scale = tensors["down_scale"].to(torch.float16).cpu()
+        self.alpha = float(alpha)
+
+    def materialize(self, device: torch.device, dtype: torch.dtype) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        gate = self.gate_ternary.to(device=device, dtype=dtype) * self.gate_scale.to(device=device, dtype=dtype).unsqueeze(-1)
+        up = self.up_ternary.to(device=device, dtype=dtype) * self.up_scale.to(device=device, dtype=dtype).unsqueeze(-1)
+        down = self.down_ternary.to(device=device, dtype=dtype) * self.down_scale.to(device=device, dtype=dtype).unsqueeze(-1)
+        return gate, up, down
+
+
+class EvalRoutedQuantizedMoE(nn.Module):
+    def __init__(self, shared_mlp: nn.Module, experts: dict[int, CPUQuantizedExpert], router_weight: torch.Tensor, *, top_k: int) -> None:
+        super().__init__()
+        self.shared_mlp = shared_mlp
+        self.experts = experts
+        self.register_buffer("router_weight", router_weight.float().contiguous(), persistent=False)
+        self.top_k = int(top_k)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        shared_out = self.shared_mlp(x)
+        batch, seq_len, hidden = x.shape
+        x_flat = x.reshape(-1, hidden)
+        shared_flat = shared_out.reshape(-1, hidden)
+        router_weight = self.router_weight.to(device=x.device, dtype=torch.float32)
+        logits = F.linear(x_flat.float(), router_weight)
+        vals, idx = torch.topk(logits, k=min(self.top_k, router_weight.shape[0]), dim=-1)
+        weights = F.softmax(vals, dim=-1)
+        mixed = shared_flat.float()
+        target_dtype = x.dtype if x.dtype in (torch.float16, torch.bfloat16) else torch.float32
+
+        for expert_idx, expert in self.experts.items():
+            token_idx, choice_idx = torch.where(idx == expert_idx)
+            if token_idx.numel() == 0:
+                continue
+            gate_w, up_w, down_w = expert.materialize(x.device, target_dtype)
+            x_tok = x_flat[token_idx].to(dtype=target_dtype)
+            gate = F.linear(x_tok, gate_w)
+            up = F.linear(x_tok, up_w)
+            out = F.linear(F.silu(gate) * up, down_w)
+            delta = out.float() - shared_flat[token_idx].float()
+            mixed[token_idx] += weights[token_idx, choice_idx].unsqueeze(-1) * expert.alpha * delta
+            del gate_w, up_w, down_w, x_tok, gate, up, out, delta
+
+        return mixed.to(dtype=shared_out.dtype).reshape(batch, seq_len, hidden)
+
+
+def _load_layer_experts(model_dir: Path, layer_idx: int, experts_per_layer: int, alpha_map: dict[int, dict[int, float]]) -> dict[int, CPUQuantizedExpert]:
+    expert_dir = model_dir / "experts"
+    layer_alphas = alpha_map.get(layer_idx, {})
+    experts: dict[int, CPUQuantizedExpert] = {}
+    for expert_idx in range(experts_per_layer):
+        path = expert_dir / f"layer_{layer_idx:02d}_expert_{expert_idx:02d}.safetensors"
+        if not path.exists():
+            continue
+        with safe_open(str(path), framework="pt", device="cpu") as handle:
+            tensors = {key: handle.get_tensor(key) for key in handle.keys()}
+        experts[expert_idx] = CPUQuantizedExpert(tensors, layer_alphas.get(expert_idx, 0.0))
+    return experts
+
+
+class OutlierMoEForCausalLM(PreTrainedModel):
+    config_class = OutlierMoEConfig
+
+    def __init__(self, config: OutlierMoEConfig) -> None:
+        super().__init__(config)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, *model_args, config=None, **kwargs):
+        model_dir = Path(pretrained_model_name_or_path)
+        if config is None:
+            config = OutlierMoEConfig.from_pretrained(model_dir)
+
+        base_kwargs = {}
+        for key in ("trust_remote_code", "device_map", "low_cpu_mem_usage", "attn_implementation"):
+            if key in kwargs:
+                base_kwargs[key] = kwargs.pop(key)
+
+        torch_dtype = kwargs.pop("torch_dtype", None)
+        if torch_dtype is None and "dtype" in kwargs:
+            torch_dtype = kwargs.pop("dtype")
+        if torch_dtype is not None:
+            base_kwargs["torch_dtype"] = _parse_dtype(torch_dtype)
+
+        model = AutoModelForCausalLM.from_pretrained(
+            config.base_model_name_or_path,
+            **base_kwargs,
+        )
+
+        alpha_map = _load_alpha_map(model_dir)
+        router_map = _load_router_map(model_dir)
+        layers = list(getattr(config, "moe_layers", []))
+        experts_per_layer = int(getattr(config, "n_experts", 0))
+        top_k = int(getattr(config, "top_k", 2))
+
+        for layer_idx in layers:
+            layer = model.model.layers[layer_idx]
+            experts = _load_layer_experts(model_dir, layer_idx, experts_per_layer, alpha_map)
+            router_weight = router_map[layer_idx]
+            layer.mlp = EvalRoutedQuantizedMoE(layer.mlp, experts, router_weight, top_k=top_k)
+
+        model.config = config
+        return model

router_state.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d1ecc6089f24fce21acaebe83a52bfa9ca3122e4b0f66044d7968b6a542da44
+size 4262200

tokenizer_config.json

@@ -0,0 +1,207 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_summary.json

@@ -0,0 +1,68 @@
+{
+  "status": "complete",
+  "base_model": "/mnt/1tb/Qwen2.5-14B-Instruct",
+  "output_dir": "/mnt/1tb/outlier-40b-v3_2",
+  "moe_layers": [
+    11,
+    12,
+    13,
+    14,
+    15,
+    16,
+    17,
+    18,
+    19,
+    20,
+    21,
+    22,
+    23,
+    24,
+    25,
+    26,
+    27,
+    28,
+    29,
+    30,
+    31,
+    32,
+    33,
+    34,
+    35,
+    36
+  ],
+  "experts_per_layer": 8,
+  "parallel_experts": 8,
+  "steps_per_expert": 1500,
+  "batch_size": 4,
+  "lr": 0.001,
+  "router_steps": 100,
+  "router_lr": 0.001,
+  "top_k_logits": 128,
+  "topk_experts": 2,
+  "corpus_examples": 5000,
+  "elapsed_min": 368.89529099485,
+  "peak_gpu_gb": 98.52585554122925,
+  "router_summary": {
+    "router_loss": 0.050282299518585205,
+    "router_time_s": 443.438959875999,
+    "peak_gpu_gb": 98.52585554122925,
+    "chunks": 7
+  },
+  "diversity": {
+    "avg_pairwise_cosine": 0.9031525016813488,
+    "min_pairwise_cosine": 0.6628175973892212,
+    "avg_ternary_zero_rate": 0.3134468748019292
+  },
+  "alpha_stats": {
+    "min": 0.007545572705566883,
+    "max": 0.09620270878076553,
+    "mean": 0.06583456943133989
+  },
+  "memory_estimate": {
+    "expert_state_gb": 3.1640625,
+    "rough_base_gb": 28.0,
+    "rough_overhead_gb": 45.0,
+    "rough_total_gb": 98.3125
+  },
+  "training_records": 39000
+}