diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..e2bfbc7d2ad28de991fabe81ff53da7d0f226f15 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.safetensors filter=lfs diff=lfs merge=lfs -text +assets/k2-horizon-mova-36b-a4b-benchmarks.png filter=lfs diff=lfs merge=lfs -text +tokenizer.json filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..535d1cd6fdf13ed79dd9c485b6a794661886aad6 --- /dev/null +++ b/README.md @@ -0,0 +1,56 @@ +--- +base_model: IFM/K2-Horizon-MoVA-36B-A4B +base_model_relation: quantized +license: apache-2.0 +language: + - en +library_name: mlx +pipeline_tag: text-generation +tags: + - mlx + - mlx-lm + - 4-bit + - k2-horizon + - long-context + - 512k-context + - mixture-of-experts + - mova +--- + +# K2-Horizon-MoVA-36B-A4B MLX-4bit + +MLX 4-bit conversion of [IFM/K2-Horizon-MoVA-36B-A4B](https://huggingface.co/IFM/K2-Horizon-MoVA-36B-A4B), a sparse Mixture-of-Experts model with Mixture-of-Values attention (MoVA), 36B total parameters, and approximately 4B active parameters per token. The source checkpoint supports a native context length of **524,288 tokens (512K)**. + +## Benchmarks + +![K2-Horizon-MoVA-36B-A4B benchmark results](assets/k2-horizon-mova-36b-a4b-benchmarks.png) + +*Benchmark results reported by IFM for the original K2-Horizon-MoVA-36B-A4B checkpoint.* + +## Release + +| Format | Quantization | Size | +| --- | --- | ---: | +| MLX safetensors | Affine 4-bit, group size 64 | 21.07 GB | + +The MoE and MoVA router weights remain at 8-bit; other eligible weights use affine 4-bit quantization. This is a text-only release; no vision projector is required. The repository includes a source-compatible [`chat_template.jinja`](chat_template.jinja), the custom MLX model implementation required by this architecture, and [`SHA256SUMS`](SHA256SUMS). + +## Usage + +```bash +pip install -U mlx-lm + +mlx_lm.generate \ + --model abenzerps/K2-Horizon-MoVA-36B-A4B-MLX-4bit \ + --prompt "Explain why reproducible builds matter." \ + --max-tokens 512 --temp 1.0 --top-p 0.95 +``` + +Use a current `mlx-lm` release. The source model supports 512K context; usable context length depends on available unified memory and KV-cache settings. + +## Source + +- Model: [IFM/K2-Horizon-MoVA-36B-A4B](https://huggingface.co/IFM/K2-Horizon-MoVA-36B-A4B) +- Source revision: [`05cab0a`](https://huggingface.co/IFM/K2-Horizon-MoVA-36B-A4B/commit/05cab0a4d7150c1c460a000b37ff40cc1af2feaa) +- License: [Apache-2.0](https://www.apache.org/licenses/LICENSE-2.0) +- Checksums: [SHA256SUMS](SHA256SUMS) diff --git a/SHA256SUMS b/SHA256SUMS new file mode 100644 index 0000000000000000000000000000000000000000..0254e8746c1023c490c6cb64eaaec662768bc475 --- /dev/null +++ b/SHA256SUMS @@ -0,0 +1,58 @@ +6ac63ccc18056fdfa10253e0ef3f510e9c14da1cdc7042ba04d3a8f10e0d2a97 ./README.md +6360bd274549f0b40d732d10f91553e3e4335732c7c2f6a53eac6cf67c30116d ./assets/k2-horizon-mova-36b-a4b-benchmarks.png +a892cd0b0195599f283a8c706787520d9a6747640efb2f4dec4144b0abb62590 ./chat_template.jinja +6cc0fa8d04e2feacbb7a885160b6b97c9b56df13531b85b3d2a153d830cba176 ./config.json +2da7d47641f4509da4ae47711e31d8b5f0f3f801ee08d87e7e9f07f814bdc4a3 ./generation_config.json +0b0417dd7430467a7847cbd0611ad09d45a0a91ce86720022e8cc00aa1cf42ef ./k2_horizon_mova_mlx.py +31e5235971c9831d85f84ffa50bf5fe4bebc410c99aac3026073b231c64fdeee ./model-00001-of-00048.safetensors +9d08fb67b1a30b7150e73eecdce19a76d0fd273a735bd318d32242070ca71612 ./model-00002-of-00048.safetensors +1bd9d907b1104667c2f5572d451823c9fed5229d7b7b411b19b1ca56ed71863a ./model-00003-of-00048.safetensors +fcf17a50a793a7d6c75efa7890cd45177e1dff6dcf87934075ca236abd5d82e5 ./model-00004-of-00048.safetensors +c60a0c455f5d866944a8ab1d1b6b31ec11f0e83b3333365f9cf6ffbbf2541684 ./model-00005-of-00048.safetensors +51401f4e04332968247f3d1525359647bb5f810e457f577772a5c7d8510bceef ./model-00006-of-00048.safetensors +50a7ea11975014ee47cdbbc1a154e2bb8167ea0303c943d60c6a91804efdc737 ./model-00007-of-00048.safetensors +9558e7e8f36cc59073dddb1e9493b04b61dafee961f0bf7ac37d9ba171558dba ./model-00008-of-00048.safetensors +6636e298689e80a83668d798170e5a9f2988cbc97684f220319001be98b3bec1 ./model-00009-of-00048.safetensors +fc7d03e8f8841b5eb7bc8884a2e244c09aea5f29ee136837614bafdf939cb042 ./model-00010-of-00048.safetensors +3308837f6d5eed5bc980446b69f88df0444d9041e4cfc86e495a9a6b776dc733 ./model-00011-of-00048.safetensors +30c4d8f41d8586da70e3d6a2f193cea93b7b54ee5eff8a647eebb42c223e6ee9 ./model-00012-of-00048.safetensors +53302fc7cf9f635ad3b536df999f1a30a2ac9b6bbd34981473d3e07a8924fbbf ./model-00013-of-00048.safetensors +33a6bd51e600e5f767d7cc7b898bf958e8fb14573fa76f204e9b3a461d3d1da1 ./model-00014-of-00048.safetensors +7140efd36587a70c9984adc0aa00523951d0a3b9feda24134faa4bccc93a8457 ./model-00015-of-00048.safetensors +9371d749bc50601fccf32d6ed497fc9e3c6cf8f228cabbe9ed847a4c2bd69840 ./model-00016-of-00048.safetensors +027c61bb1e5ea9582bc6fb9349b9e5f1633f47db1ee4635d1be6600d95b39c85 ./model-00017-of-00048.safetensors +f6dbe8d96f1c83389dc96a4a2fc7722924c48b63e3959cd3aaa771a7b1ba2e54 ./model-00018-of-00048.safetensors +455100cce3c5be76660c62d368e44491c963e7bbb10062cf85cfacf886753751 ./model-00019-of-00048.safetensors +e72327b996a2fad2198d21b49afb2850c3644ec372847525156604029d2a183a ./model-00020-of-00048.safetensors +3b8c5f6ea0b6207283dc02df0b20b23f462fd4a68d8716a2e74a1cb69786c452 ./model-00021-of-00048.safetensors +d4ec0318b4e56e797be38ae57b5c72b65f9ab681179f2368467493aa556a74ca ./model-00022-of-00048.safetensors +1b8c0abcaae152db6d66ae069c29d81476b9786c9b9d485ebf3705fbe39ef7df ./model-00023-of-00048.safetensors +db2f9bfb742391e8ed2d53bb83ba78ab0b5225580ccb6018b8c0d8e5bd4fb0e0 ./model-00024-of-00048.safetensors +70f63ca25cc527417a1cad79cf58bcb5616f35f97f4027d0fddeb9ad5196438b ./model-00025-of-00048.safetensors +89b205e4659789bfd58bf49004a76b4fa99d9bbdd1fcae5ec861ea4472677627 ./model-00026-of-00048.safetensors +209af642e45016b37be905086e163b4ca111dd33cf48897317ed7b243e8c970e ./model-00027-of-00048.safetensors +bd11b46f53a8c83da2f2311309065e33634235001a236e1c717a65f201029af7 ./model-00028-of-00048.safetensors +461c72b2368caa565d95613d9f707f18621a7a2b76b5ddbedfbdbe37589d8c80 ./model-00029-of-00048.safetensors +3a2ed651142e80d299978a6c1caabd4e23fb5650e7ef3e11bfec967d921ffa97 ./model-00030-of-00048.safetensors +6dc06c4d8b90ceb7461ed5bddd9134a08c05695972c00e66d38599a2efe7b894 ./model-00031-of-00048.safetensors +c25408fc0a757c47128efbfed6cfd1faeca4641c79931461e0b0385c83eec80e ./model-00032-of-00048.safetensors +5e35e950738d0e9fa5e272d87b91d6368b0c90a73b1aa378c1144cc28fabdd5a ./model-00033-of-00048.safetensors +ce9adbb1346a4bb7049e96ceff3ca7b68c2d8550ffbabee69b5bf45c5e6f0564 ./model-00034-of-00048.safetensors +b2c64688e850c215c39759a030ab03acad66e6953b82e75be7ed86563893d5ec ./model-00035-of-00048.safetensors +6b182ec03b2aee2fe6a8abda92a0da562839299833a1228d8ccf0c05e23125cc ./model-00036-of-00048.safetensors +7e2721a914cd08404fb97f3cc5a1ac2b637c649e517db2bbb4e2b969dc5eee8f ./model-00037-of-00048.safetensors +350aa2c893d468fdb7bd38f5ae026f230198654f48bd57e14b4b00bce5ad94c8 ./model-00038-of-00048.safetensors +c9491cd273404aad9d152183969870833ac785016938f76abcf40ffca02609a3 ./model-00039-of-00048.safetensors +c0692877de523630bb33b782d71f690abce2f8af23b3357ec817b3bf2a4d5de2 ./model-00040-of-00048.safetensors +e6502976c7222801e6839aef84c666b4766aecc8fb42680804c1ee7b5fccfeb5 ./model-00041-of-00048.safetensors +61618b4716835ec1e3405163a9def07e70745dcb1bd1583c783b655b1b35d732 ./model-00042-of-00048.safetensors +755485d662cb6e52916f1e00e27e0548bb46ea1363945f296224ea1ede50681c ./model-00043-of-00048.safetensors +f93a315a0732690c3dbbe80dfd32c23a1eb2d930c075b2f4ae7dae11134760ec ./model-00044-of-00048.safetensors +b53e9662206809c13ea095de8d324a3acfe53b35f9cccc4baa8b22c39ffdff26 ./model-00045-of-00048.safetensors +346ffb61c16963d6e15fe685e848155bf7f92f5e23a97cb15b5e8f62713b48d5 ./model-00046-of-00048.safetensors +d89a0b584c6ba903950544f62bae40f5d503e734e574d892402fa6ed34783f73 ./model-00047-of-00048.safetensors +bfc24affec2ce026fe5ce6036812689866b3efc17716d23ec602e67ae4111f63 ./model-00048-of-00048.safetensors +92e2d5c7f72249d8992e982eafab2316ceae5a09e71784445c0b6219a332d426 ./model.safetensors.index.json +e9be461abfcb63da71fb65db470e075682e7573e469cc658aeeb6632a6144ae1 ./special_tokens_map.json +2fa69519ff1ee3ee55ac50927b00058e88b9eb90802fabaf3c6b05804601c53b ./tokenizer.json +581375b2ce8d56d59f609f08a7165365ac53b6543f3442ef0117b4951b0d747c ./tokenizer_config.json diff --git a/assets/k2-horizon-mova-36b-a4b-benchmarks.png b/assets/k2-horizon-mova-36b-a4b-benchmarks.png new file mode 100644 index 0000000000000000000000000000000000000000..5b17f9e7545bfef210d449526abdbbc446ad7e98 --- /dev/null +++ b/assets/k2-horizon-mova-36b-a4b-benchmarks.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6360bd274549f0b40d732d10f91553e3e4335732c7c2f6a53eac6cf67c30116d +size 885256 diff --git a/chat_template.jinja b/chat_template.jinja new file mode 100644 index 0000000000000000000000000000000000000000..41dde1f4bc64d0554f7c526207ff618be6c10ff0 --- /dev/null +++ b/chat_template.jinja @@ -0,0 +1,994 @@ +{{- bos_token }} +{%- if tool_presentation is defined -%} + {{- raise_exception("Unsupported argument: tool_presentation. Use tool_presentation_format with one of: json, xml, markdown.") -}} +{%- endif -%} +{%- if tool_calling_format is defined -%} + {{- raise_exception("Unsupported argument: tool_calling_format. Use tool_call_format with one of: json, xml, xml_typed.") -}} +{%- endif -%} +{%- if tool_format is defined -%} + {{- raise_exception("Unsupported argument: tool_format. Use tool_call_format with one of: json, xml, xml_typed.") -}} +{%- endif -%} +{%- set tool_presentation_fmt = tool_presentation_format | default('markdown') -%} +{%- set tool_call_fmt = tool_call_format | default('xml') -%} +{%- if tool_presentation_fmt != 'json' and tool_presentation_fmt != 'xml' and tool_presentation_fmt != 'markdown' -%} + {{- raise_exception("Unsupported tool_presentation_format: '" ~ tool_presentation_fmt ~ "'. Supported formats: json, xml, markdown.") -}} +{%- endif -%} +{%- if tool_call_fmt != 'json' and tool_call_fmt != 'xml' and tool_call_fmt != 'xml_typed' -%} + {{- raise_exception("Unsupported tool_call_format: '" ~ tool_call_fmt ~ "'. Supported formats: json, xml, xml_typed.") -}} +{%- endif -%} + +{#- Renderability state, computed during validate_tools (single walk, no extra -#} +{#- traversal at render time): ok = working flag for the tool being validated; -#} +{#- bad = pipe-delimited indices of tools that must render as verbatim JSON. -#} +{%- set RB = namespace(ok=true, bad='|') -%} + +{%- macro value_contains_mapping(v) -%} +{%- if v is mapping -%} +true +{%- elif v is sequence and v is not string -%} +{%- set f = namespace(x='false') -%} +{%- for c in v -%}{%- if value_contains_mapping(c) == 'true' -%}{%- set f.x = 'true' -%}{%- endif -%}{%- endfor -%} +{{- f.x -}} +{%- else -%} +false +{%- endif -%} +{%- endmacro -%} + +{#- $ref inlining state: defs = local $defs of the tool being rendered; seen = -#} +{#- pipe-delimited names already expanded for this tool (each def inlines at most -#} +{#- once; later references render by def name; cycles terminate immediately). -#} +{#- $ref-sibling annotations (description/default/...) merge OVER the def at -#} +{#- the inline site, so use-site annotations win and are never dropped. -#} +{%- set REFS = namespace(defs={}, seen='|') -%} + +{%- macro render_compact_type_name(type_name, spec) -%} +{%- if type_name == "array" -%} +array[{%- if 'items' in spec -%}{{ render_compact_type(spec['items']) }}{%- else -%}any{%- endif -%}] +{%- elif type_name -%} +{{- type_name -}} +{%- else -%} +any +{%- endif -%} +{%- endmacro -%} + +{%- macro render_compact_type(spec) -%} +{%- if spec is not mapping -%} +any +{%- elif spec.type is defined and spec.type is sequence and spec.type is not string and spec.type | length > 0 -%} +{%- for type_name in spec.type -%}{{ render_compact_type_name(type_name, spec) }}{%- if not loop.last -%}|{%- endif -%}{%- endfor -%} +{%- elif spec.type is defined and spec.type is sequence and spec.type is not string -%} +any +{%- elif spec.type -%} +{{- render_compact_type_name(spec.type, spec) -}} +{%- elif spec['$ref'] is string -%} +{{- spec['$ref'].split('/') | last -}} +{%- elif spec.oneOf -%} +oneOf[{%- for variant in spec.oneOf -%}{{ render_compact_type(variant) }}{%- if not loop.last -%}|{%- endif -%}{%- endfor -%}] +{%- elif spec.anyOf -%} +anyOf[{%- for variant in spec.anyOf -%}{{ render_compact_type(variant) }}{%- if not loop.last -%}|{%- endif -%}{%- endfor -%}] +{%- elif spec.properties -%} +object +{%- elif 'items' in spec -%} +array[{{ render_compact_type(spec['items']) }}] +{%- else -%} +any +{%- endif -%} +{%- endmacro -%} + +{%- macro render_markdown_type_name(type_name, spec) -%} +{%- if type_name == "array" -%} +array of {% if 'items' in spec %}{{ render_markdown_type(spec['items']) }}{% else %}any{% endif %} +{%- elif type_name -%} +{{- type_name -}} +{%- else -%} +any +{%- endif -%} +{%- endmacro -%} + +{%- macro render_markdown_type(spec) -%} +{%- if spec is sameas true -%} +True +{%- elif spec is sameas false -%} +False +{%- elif spec is not mapping -%} +any +{%- elif spec.type is defined and spec.type is sequence and spec.type is not string and spec.type | length > 0 -%} +{%- for type_name in spec.type -%}{{ render_markdown_type_name(type_name, spec) }}{% if not loop.last %} or {% endif %}{%- endfor -%} +{%- elif spec.type is defined and spec.type is sequence and spec.type is not string -%} +any +{%- elif spec.type -%} +{{- render_markdown_type_name(spec.type, spec) -}} +{%- elif spec['$ref'] is string -%} +{{- spec['$ref'].split('/') | last -}} +{%- elif spec.oneOf -%} +oneOf[{%- for variant in spec.oneOf -%}{{ render_markdown_type(variant) }}{% if not loop.last %} or {% endif %}{%- endfor -%}] +{%- elif spec.anyOf -%} +anyOf[{%- for variant in spec.anyOf -%}{{ render_markdown_type(variant) }}{% if not loop.last %} or {% endif %}{%- endfor -%}] +{%- elif spec.properties -%} +object +{%- elif 'items' in spec -%} +array of {{ render_markdown_type(spec['items']) }} +{%- else -%} +any +{%- endif -%} +{%- endmacro -%} + +{%- macro render_xml_text(value) -%} +{{- value.split() | join(" ") -}} +{%- endmacro -%} + +{%- macro render_python_string(value) -%} +'{{- value.split() | join(" ") | replace("\\", "\\\\") | replace("'", "\\'") -}}' +{%- endmacro -%} + +{%- macro render_python_repr(value) -%} +{%- if value is string -%} +{{ render_python_string(value) }} +{%- elif value is sameas true -%} +True +{%- elif value is sameas false -%} +False +{%- elif value is none -%} +None +{%- elif value is mapping -%} +{{- "{" -}} +{%- for key, child in value | items -%} +{{ render_python_repr(key) }}: {{ render_python_repr(child) }}{%- if not loop.last -%}, {% endif -%} +{%- endfor -%} +{{- "}" -}} +{%- elif value is sequence -%} +{{- "[" -}} +{%- for child in value -%} +{{ render_python_repr(child) }}{%- if not loop.last -%}, {% endif -%} +{%- endfor -%} +{{- "]" -}} +{%- else -%} +{{- value -}} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_xml_value(value) -%} +{%- if value is string -%}{{ render_xml_text(value) }}{%- else -%}{{ render_python_repr(value) }}{%- endif -%} +{%- endmacro -%} + +{%- macro render_xml_enum_value(value) -%} +{%- if value is string -%}"{{- value | replace("\\", "\\\\") | replace("\"", "\\\"") -}}"{%- else -%}"{{- render_python_repr(value) | replace("\\", "\\\\") | replace("\"", "\\\"") -}}"{%- endif -%} +{%- endmacro -%} + +{%- macro render_xml_enum(values) -%} +{%- for value in values -%}{{ render_xml_enum_value(value) }}{%- if not loop.last -%}|{%- endif -%}{%- endfor -%} +{%- endmacro -%} + +{%- macro render_xml_default_attr(value) -%} +{{- " default=" }}{%- if value is string -%}"{{- value | replace("\\", "\\\\") | replace("\"", "\\\"") -}}"{%- else -%}{{ render_xml_value(value) }}{%- endif -%} +{%- endmacro -%} + +{%- macro render_xml_attr(name, value) -%} +{{- " " + name + "=" }}{%- if value == "" -%}""{%- else -%}{{ render_xml_value(value) }}{%- endif -%} +{%- endmacro -%} + +{%- macro validate_schema(spec, path, lenient=false, classify=true, in_variant=false) -%} +{%- if spec is mapping -%} + {%- if not lenient -%} + {%- if spec.required is defined -%} + {%- if spec.required is string or spec.required is not sequence -%} + {{- raise_exception("Schema '" + path + "' has 'required' but it is not a list.") -}} + {%- endif -%} + {%- if spec.required | length > 0 and not spec.properties and not in_variant -%} + {{- raise_exception("Schema '" + path + "' has required fields but no properties object to define them.") -}} + {%- endif -%} + {%- if spec.properties -%} + {%- for required_name in spec.required -%} + {%- if required_name not in spec.properties -%} + {{- raise_exception("Schema '" + path + "' marks '" + required_name + "' as required, but that property is not defined in properties.") -}} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- endif -%} + {%- endif -%} + {#- renderability classification, piggybacking on this walk (no raises here): -#} + {#- constructs the pretty renderer does not fully handle flip RB.ok so the -#} + {#- tool falls back to verbatim JSON. Skipped entirely for json presentation. -#} + {%- if classify -%} + {%- for key, value in spec | items -%} + {%- if key == '$ref' -%} + {%- if value is not string -%}{%- set RB.ok = false -%} + {%- elif not (value.startswith('#/$defs/') or value.startswith('#/definitions/')) -%}{%- set RB.ok = false -%}{%- endif -%} + {%- elif key == '$defs' or key == 'definitions' -%} + {%- if value is mapping -%} + {%- for dk, dv in value | items -%} + {{- validate_schema(dv, path + ".$defs." + dk, true) -}} + {%- endfor -%} + {%- else -%}{%- set RB.ok = false -%}{%- endif -%} + {%- elif key == 'type' -%} + {%- if value is mapping -%}{%- set RB.ok = false -%}{%- endif -%} + {%- elif key == 'enum' -%} + {%- if value is string or value is mapping or value is not sequence -%}{%- set RB.ok = false -%}{%- endif -%} + {%- elif key == 'items' -%} + {#- any items shape renders: mapping structurally, others via repr detail -#} + {%- elif key == 'oneOf' or key == 'anyOf' -%} + {%- if value is mapping or value is string or value is not sequence -%}{%- set RB.ok = false -%}{%- endif -%} + {%- elif key == 'required' -%} + {%- if value and not spec.properties -%}{%- set RB.ok = false -%}{%- endif -%} + {%- elif ('|' ~ key ~ '|') in '|description|default|title|examples|properties|patternProperties|additionalProperties|returns|' -%} + {%- elif value is mapping -%} + {%- for uk, uv in value | items -%} + {%- if value_contains_mapping(uv) == 'true' -%}{%- set RB.ok = false -%}{%- endif -%} + {%- endfor -%} + {%- elif value is sequence and value is not string -%} + {%- if value_contains_mapping(value) == 'true' -%}{%- set RB.ok = false -%}{%- endif -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- if spec.properties -%} + {%- for child_name, child_spec in spec.properties | items -%} + {{- validate_schema(child_spec, path + "." + child_name, lenient, classify) -}} + {%- endfor -%} + {%- endif -%} + {%- if 'items' in spec -%}{{- validate_schema(spec['items'], path + "[]", lenient, classify) -}}{%- endif -%} + {%- if spec.oneOf -%} + {%- for variant in spec.oneOf -%}{{- validate_schema(variant, path + ".oneOf[" + (loop.index0 | string) + "]", lenient, classify, true) -}}{%- endfor -%} + {%- endif -%} + {%- if spec.anyOf -%} + {%- for variant in spec.anyOf -%}{{- validate_schema(variant, path + ".anyOf[" + (loop.index0 | string) + "]", lenient, classify, true) -}}{%- endfor -%} + {%- endif -%} + {%- if spec.additionalProperties is mapping -%}{{- validate_schema(spec.additionalProperties, path + ".additionalProperties", lenient, classify) -}}{%- endif -%} + {%- if spec.patternProperties is mapping -%} + {%- for pattern, pattern_spec in spec.patternProperties | items -%} + {{- validate_schema(pattern_spec, path + ".patternProperties[" + pattern + "]", lenient, classify) -}} + {%- endfor -%} + {%- endif -%} + {%- if spec.returns is mapping -%}{{- validate_schema(spec.returns, path + ".returns", lenient, classify) -}}{%- endif -%} +{%- endif -%} +{%- endmacro -%} + +{%- macro validate_tools(tools_list, classify=true) -%} +{%- set RB.bad = '|' -%} +{%- for tool in tools_list -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set RB.ok = true -%} + {%- if fn.parameters is defined and fn.parameters is string -%} + {{- raise_exception("tool.function.parameters must be a dict, not a JSON string. Parse it before passing to the template.") -}} + {%- endif -%} + {%- if fn.parameters is not defined or fn.parameters is none -%} + {%- if fn.arguments is defined -%} + {{- raise_exception("Tool '" + fn.name + "' has 'arguments' instead of 'parameters'. Rename 'arguments' to 'parameters'.") -}} + {%- else -%} + {{- raise_exception("Tool '" + fn.name + "' is missing required 'parameters' field. Each tool must have a 'parameters' dict with 'type', 'properties', and 'required' keys.") -}} + {%- endif -%} + {%- endif -%} + {{- validate_schema(fn.parameters, "tool." + fn.name + ".parameters", false, classify) -}} + {%- if classify -%} + {%- if fn.parameters is mapping -%} + {#- unknown container-valued keys at the parameters ROOT are never rendered -#} + {#- by the pretty path (root extras are dropped) -> verbatim fallback. -#} + {%- for rk, rv in fn.parameters | items -%} + {%- if rk not in ['type', 'description', 'enum', 'default', 'properties', 'required', 'optional', 'title', 'items', 'oneOf', 'anyOf', 'additionalProperties', 'patternProperties', 'returns', 'examples', '$defs', 'definitions', '$ref'] -%} + {%- if rv is mapping or (rv is sequence and rv is not string) -%}{%- set RB.ok = false -%}{%- endif -%} + {%- endif -%} + {%- endfor -%} + {%- else -%} + {%- set RB.ok = false -%} + {%- endif -%} + {%- endif -%} + {%- if fn.returns is mapping -%}{{- validate_schema(fn.returns, "tool." + fn.name + ".returns", false, classify) -}}{%- endif -%} + {%- if classify and fn.returns is not defined and fn.response is mapping -%}{{- validate_schema(fn.response, "tool." + fn.name + ".response", true) -}}{%- endif -%} + {#- unknown container-valued keys at the FUNCTION level are never rendered -> fallback. -#} + {%- if classify -%} + {%- for fk, fv in fn | items -%} + {%- if fk not in ['name', 'description', 'parameters', 'returns', 'response', 'type', 'function'] -%} + {%- if fv is mapping or (fv is sequence and fv is not string) -%}{%- set RB.ok = false -%}{%- endif -%} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + {%- if not RB.ok -%}{%- set RB.bad = RB.bad ~ loop.index0 ~ '|' -%}{%- endif -%} +{%- endfor -%} +{%- endmacro -%} + +{%- macro render_tools_json(tools_list) -%} +{{- "" }} +{%- for tool in tools_list %} +{{- "\n" }} +{{- tool | tojson }} +{%- endfor %} +{{- "\n" }} +{%- endmacro -%} + +{%- macro render_xml_schema_attrs(spec, include_value_attrs) -%} +{%- if spec is mapping -%} +{%- set structural_keys = ["type", "description", "enum", "default", "properties", "required", "items", "oneOf", "anyOf", "additionalProperties", "patternProperties", "returns"] -%} +{%- if include_value_attrs and spec.enum -%}{{- " enum=" }}{{ render_xml_enum(spec.enum) }}{%- endif -%} +{%- if include_value_attrs and spec.default is defined -%}{{ render_xml_default_attr(spec.default) }}{%- endif -%} +{%- if spec.additionalProperties is defined and spec.additionalProperties is not mapping -%}{{ render_xml_attr("additionalProperties", spec.additionalProperties) }}{%- endif -%} +{%- if spec.patternProperties is defined and spec.patternProperties is not mapping -%}{{ render_xml_attr("patternProperties", spec.patternProperties) }}{%- endif -%} +{%- for key, value in spec | items -%} + {%- if key not in structural_keys -%} +{{ render_xml_attr(key, value) }} + {%- endif -%} +{%- endfor -%} +{%- endif -%} +{%- endmacro -%} + +{%- macro xml_schema_has_children(spec, include_properties, include_description) -%} +{%- if spec is not mapping -%} +false +{%- elif (include_description and spec.description is defined) or (include_properties and spec.properties) or 'items' in spec or spec.oneOf or spec.anyOf or spec.additionalProperties is mapping or spec.patternProperties is mapping or spec.returns is defined -%} +true +{%- else -%} +false +{%- endif -%} +{%- endmacro -%} + +{%- macro render_xml_schema_node(tag, spec, include_properties) -%} +{%- if spec is mapping and spec['$ref'] is string -%} + {%- set _r = spec['$ref'] -%} + {%- set _k = _r[8:] if _r.startswith('#/$defs/') else (_r[14:] if _r.startswith('#/definitions/') else none) -%} + {%- if _k is not none and ('|' + _k + '|') not in REFS.seen and REFS.defs[_k] is mapping -%} + {%- set spec = dict((REFS.defs[_k] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k + '|' -%} + {%- if spec['$ref'] is string -%} + {%- set _r2 = spec['$ref'] -%} + {%- set _k2 = _r2[8:] if _r2.startswith('#/$defs/') else (_r2[14:] if _r2.startswith('#/definitions/') else none) -%} + {%- if _k2 is not none and REFS.defs[_k2] is mapping -%} + {%- set spec = dict((REFS.defs[_k2] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k2 + '|' -%} + {%- endif -%} + {%- endif -%} + {%- endif -%} +{%- endif -%} +{%- if spec is mapping -%} +{{- "<" + tag + " type=" + render_compact_type(spec) }}{{ render_xml_schema_attrs(spec, true) }} +{%- if xml_schema_has_children(spec, include_properties, true) == 'true' -%} +{{- ">" }}{{ render_xml_schema_children(spec, include_properties, true) }}{{- "" }} +{%- else -%} +{{- "/>" }} +{%- endif -%} +{%- else -%} +{{- "<" + tag + ">" }}{{ render_xml_value(spec) }}{{- "" }} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_xml_pattern_property(pattern, spec) -%} +{%- if spec is mapping and spec['$ref'] is string -%} + {%- set _r = spec['$ref'] -%} + {%- set _k = _r[8:] if _r.startswith('#/$defs/') else (_r[14:] if _r.startswith('#/definitions/') else none) -%} + {%- if _k is not none and ('|' + _k + '|') not in REFS.seen and REFS.defs[_k] is mapping -%} + {%- set spec = dict((REFS.defs[_k] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k + '|' -%} + {%- if spec['$ref'] is string -%} + {%- set _r2 = spec['$ref'] -%} + {%- set _k2 = _r2[8:] if _r2.startswith('#/$defs/') else (_r2[14:] if _r2.startswith('#/definitions/') else none) -%} + {%- if _k2 is not none and REFS.defs[_k2] is mapping -%} + {%- set spec = dict((REFS.defs[_k2] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k2 + '|' -%} + {%- endif -%} + {%- endif -%} + {%- endif -%} +{%- endif -%} +{%- if spec is mapping -%} +{{- "" }}{{ render_xml_schema_children(spec, true, true) }}{{- "" }} +{%- else -%} +{{- "/>" }} +{%- endif -%} +{%- else -%} +{{- "" }}{{ render_xml_value(spec) }}{{- "" }} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_xml_schema_children(spec, include_properties, include_description) -%} +{%- if include_description and spec.description is defined -%}{{- "" }}{{ spec.description }}{{- "" }}{%- endif -%} +{%- if include_properties and spec.properties -%} +{%- for child_name, child_spec in spec.properties | items -%} +{{- render_xml_param(child_name, child_spec, spec.required or []) }} +{%- endfor -%} +{%- endif -%} +{%- if 'items' in spec -%}{{ render_xml_schema_node("items", spec['items'], true) }}{%- endif -%} +{%- if spec.oneOf -%} +{{- "" }} +{%- for variant in spec.oneOf -%}{{ render_xml_schema_node("variant", variant, true) }}{%- endfor -%} +{{- "" }} +{%- endif -%} +{%- if spec.anyOf -%} +{{- "" }} +{%- for variant in spec.anyOf -%}{{ render_xml_schema_node("variant", variant, true) }}{%- endfor -%} +{{- "" }} +{%- endif -%} +{%- if spec.additionalProperties is mapping -%}{{ render_xml_schema_node("additionalProperties", spec.additionalProperties, true) }}{%- endif -%} +{%- if spec.patternProperties is mapping -%} +{{- "" }} +{%- for pattern, pattern_spec in spec.patternProperties | items -%}{{ render_xml_pattern_property(pattern, pattern_spec) }}{%- endfor -%} +{{- "" }} +{%- elif spec.patternProperties is defined -%}{{ render_xml_value(spec.patternProperties) }}{%- endif -%} +{%- if spec.returns is mapping -%}{{ render_xml_schema_node("returns", spec.returns, true) }}{%- elif spec.returns is defined -%}{{ render_xml_value(spec.returns) }}{%- endif -%} +{%- endmacro -%} + +{%- macro render_xml_param(name, spec, required_list) -%} +{%- if spec is mapping and spec['$ref'] is string -%} + {%- set _r = spec['$ref'] -%} + {%- set _k = _r[8:] if _r.startswith('#/$defs/') else (_r[14:] if _r.startswith('#/definitions/') else none) -%} + {%- if _k is not none and ('|' + _k + '|') not in REFS.seen and REFS.defs[_k] is mapping -%} + {%- set spec = dict((REFS.defs[_k] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k + '|' -%} + {%- if spec['$ref'] is string -%} + {%- set _r2 = spec['$ref'] -%} + {%- set _k2 = _r2[8:] if _r2.startswith('#/$defs/') else (_r2[14:] if _r2.startswith('#/definitions/') else none) -%} + {%- if _k2 is not none and REFS.defs[_k2] is mapping -%} + {%- set spec = dict((REFS.defs[_k2] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k2 + '|' -%} + {%- endif -%} + {%- endif -%} + {%- endif -%} +{%- endif -%} +{{- "" }} +{%- if spec.description -%}{{ spec.description }}{%- endif -%} +{{- render_xml_schema_children(spec, true, false) }} +{{- "" }} +{%- else -%} +{{- "/>" }} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_tools_xml(tools_list) -%} +{{- "" }} +{%- for tool in tools_list -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set REFS.defs = fn.parameters['$defs'] if (fn.parameters is mapping and fn.parameters['$defs'] is mapping) else (fn.parameters['definitions'] if (fn.parameters is mapping and fn.parameters['definitions'] is mapping) else {}) -%} + {%- set REFS.seen = '|' -%} + {%- set fnp = namespace(p=fn.parameters) -%} + {%- if fnp.p is mapping and fnp.p['$ref'] is string -%} + {%- set _r = fnp.p['$ref'] -%} + {%- set _k = _r[8:] if _r.startswith('#/$defs/') else (_r[14:] if _r.startswith('#/definitions/') else none) -%} + {%- if _k is not none and REFS.defs[_k] is mapping -%} + {%- set fnp.p = dict((REFS.defs[_k] | items | list) + (fnp.p | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k + '|' -%} + {%- endif -%} + {%- endif -%} +{{- "\n" }} +{%- if fn.description -%} +{{- "" }}{{ fn.description }}{{- "" }} +{%- endif -%} +{{- "" }} +{%- if fnp.p and fnp.p.properties -%} + {%- for pname, pspec in fnp.p.properties | items -%} +{{- render_xml_param(pname, pspec, fnp.p.required or []) }} + {%- endfor -%} +{%- elif fnp.p is mapping and (fnp.p.oneOf or fnp.p.anyOf or 'items' in fnp.p) -%} +{{- render_xml_schema_children(fnp.p, true, false) }} +{%- endif -%} +{{- "" }} +{%- set fn_ret = fn.returns if fn.returns is defined else fn.response -%} +{%- if fn_ret is mapping -%}{{ render_xml_schema_node("returns", fn_ret, true) }}{%- elif fn_ret is defined -%}{{ render_xml_value(fn_ret) }}{%- endif -%} +{{- "" }} +{%- endfor -%} +{{- "\n" }} +{%- endmacro -%} + +{%- macro render_markdown_literal(value) -%} +{%- if value is string and value == "" -%}"" +{%- elif value is string -%}`{{ value | replace("\n", "\\n") }}` +{%- else -%}`{{ render_python_repr(value) }}` +{%- endif -%} +{%- endmacro -%} + +{%- macro render_allowed_values(values) -%} +{%- for value in values -%}{{ render_markdown_literal(value) }}{% if not loop.last %}, {% endif %}{%- endfor -%} +{%- endmacro -%} + +{%- macro render_markdown_value(value) -%} +{%- if value is string and value == "" -%}""{%- elif value is string -%}{{ value }}{%- else -%}{{ render_python_repr(value) }}{%- endif -%} +{%- endmacro -%} + +{%- macro render_markdown_detail(indent, label, value) -%} +{{- "\n" + indent + " - " + label + ": " }}{{ render_markdown_value(value) }} +{%- endmacro -%} + +{%- macro render_markdown_metadata_detail(label, value) -%} +{{- "\n- " + label + ": " }}{{ render_markdown_value(value) }} +{%- endmacro -%} + +{%- macro render_markdown_schema_annotations(spec, indent, include_value_details) -%} +{%- if include_value_details and spec.description is defined -%}{{ render_markdown_detail(indent, "Description", spec.description | replace("\n", "\n" + indent + " ")) }}{%- endif -%} +{%- if include_value_details and spec.enum is defined -%}{{- "\n" + indent + " - Allowed values: " }}{{ render_allowed_values(spec.enum) }}{%- endif -%} +{%- if include_value_details and spec.default is defined -%}{{- "\n" + indent + " - Default: " }}{{ render_markdown_literal(spec.default) }}{%- endif -%} +{%- if spec.additionalProperties is defined -%} + {%- if spec.additionalProperties is mapping -%} +{{- "\n" + indent + " - Additional properties *(" + render_markdown_type(spec.additionalProperties) + ")*" }} +{{- render_markdown_schema_details(spec.additionalProperties, indent + " ", true) }} + {%- else -%} +{{ render_markdown_detail(indent, "Additional properties", spec.additionalProperties) }} + {%- endif -%} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_markdown_metadata_annotations(spec) -%} +{%- if spec.description is defined -%}{{ render_markdown_metadata_detail("Description", spec.description | replace("\n", "\n ")) }}{%- endif -%} +{%- if spec.enum is defined -%}{{- "\n- Allowed values: " }}{{ render_allowed_values(spec.enum) }}{%- endif -%} +{%- if spec.default is defined -%}{{- "\n- Default: " }}{{ render_markdown_literal(spec.default) }}{%- endif -%} +{%- if spec.additionalProperties is defined -%} + {%- if spec.additionalProperties is mapping -%} +{{- "\n- Additional properties *(" + render_markdown_type(spec.additionalProperties) + ")*" }} +{{- render_markdown_schema_details(spec.additionalProperties, "", true) }} + {%- else -%} +{{ render_markdown_metadata_detail("Additional properties", spec.additionalProperties) }} + {%- endif -%} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_markdown_schema_extras(spec, indent) -%} +{%- set rendered_keys = ["type", "description", "enum", "default", "properties", "required", "items", "oneOf", "anyOf", "additionalProperties", "patternProperties", "returns"] -%} +{%- for key, value in spec | items -%} + {%- if key not in rendered_keys -%} +{{- "\n" + indent + " - " + key + ": " }}{{ render_markdown_value(value) }} + {%- endif -%} +{%- endfor -%} +{%- endmacro -%} + +{%- macro render_markdown_metadata_extras(spec) -%} +{%- set rendered_keys = ["type", "description", "enum", "default", "properties", "required", "items", "oneOf", "anyOf", "additionalProperties", "patternProperties", "returns"] -%} +{%- for key, value in spec | items -%} + {%- if key not in rendered_keys -%} +{{- "\n- " + key + ": " }}{{ render_markdown_value(value) }} + {%- endif -%} +{%- endfor -%} +{%- endmacro -%} + +{%- macro markdown_schema_has_extra(spec) -%} +{%- set rendered_keys = ["type", "description", "enum", "default", "properties", "required", "items", "oneOf", "anyOf", "additionalProperties", "patternProperties", "returns"] -%} +{%- set found = namespace(value='false') -%} +{%- for key, value in spec | items -%} + {%- if key not in rendered_keys -%}{%- set found.value = 'true' -%}{%- endif -%} +{%- endfor -%} +{{- found.value -}} +{%- endmacro -%} + +{%- macro markdown_parameter_schema_has_details(spec) -%} +{%- if spec.description is defined or spec.enum is defined or spec.default is defined or spec.additionalProperties is defined or spec.patternProperties is defined or 'items' in spec or spec.oneOf or spec.anyOf or spec.returns is defined or markdown_schema_has_extra(spec) == 'true' -%} +true +{%- else -%} +false +{%- endif -%} +{%- endmacro -%} + +{%- macro render_markdown_schema_structure(spec, indent, include_properties) -%} +{%- if include_properties and spec.properties -%} + {%- for child_name, child_spec in spec.properties | items -%} +{{- render_markdown_param(child_name, child_spec, spec.required or [], indent + " ") }} + {%- endfor -%} +{%- endif -%} +{%- if 'items' in spec and spec['items'] is mapping -%} +{{- "\n" + indent + " - Items *(" + render_markdown_type(spec['items']) + ")*" }} +{{- render_markdown_schema_details(spec['items'], indent + " ", true) }} +{%- elif 'items' in spec -%} +{{ render_markdown_detail(indent, "Items", spec['items']) }} +{%- endif -%} +{%- if spec.oneOf -%} +{{- "\n" + indent + " - oneOf:" }} + {%- for variant in spec.oneOf -%} +{{- "\n" + indent + " - Variant " }}{{ loop.index }}{{- " *(" + render_markdown_type(variant) + ")*" }} +{{- render_markdown_schema_details(variant, indent + " ", true) }} + {%- endfor -%} +{%- endif -%} +{%- if spec.anyOf -%} +{{- "\n" + indent + " - anyOf:" }} + {%- for variant in spec.anyOf -%} +{{- "\n" + indent + " - Variant " }}{{ loop.index }}{{- " *(" + render_markdown_type(variant) + ")*" }} +{{- render_markdown_schema_details(variant, indent + " ", true) }} + {%- endfor -%} +{%- endif -%} +{%- if spec.patternProperties is mapping -%} +{{- "\n" + indent + " - Pattern properties:" }} + {%- for pattern, pattern_spec in spec.patternProperties | items -%} + {%- if pattern_spec is mapping -%} +{{- "\n" + indent + " - `" + pattern + "` *(" + render_markdown_type(pattern_spec) + ")*" }} +{{- render_markdown_schema_details(pattern_spec, indent + " ", true) }} + {%- else -%} +{{- "\n" + indent + " - `" + pattern + "`: " }}{{ render_markdown_value(pattern_spec) }} + {%- endif -%} + {%- endfor -%} +{%- elif spec.patternProperties is defined -%} +{{ render_markdown_detail(indent, "Pattern properties", spec.patternProperties) }} +{%- endif -%} +{%- if spec.returns is mapping -%} +{{- "\n" + indent + " - Returns *(" + render_markdown_type(spec.returns) + ")*" }} +{{- render_markdown_schema_details(spec.returns, indent + " ", true) }} +{%- elif spec.returns is defined -%} +{{ render_markdown_detail(indent, "Returns", spec.returns) }} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_markdown_schema_details(spec, indent, include_value_details) -%} +{%- if spec is mapping and spec['$ref'] is string -%} + {%- set _r = spec['$ref'] -%} + {%- set _k = _r[8:] if _r.startswith('#/$defs/') else (_r[14:] if _r.startswith('#/definitions/') else none) -%} + {%- if _k is not none and ('|' + _k + '|') not in REFS.seen and REFS.defs[_k] is mapping -%} + {%- set spec = dict((REFS.defs[_k] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k + '|' -%} + {%- if spec['$ref'] is string -%} + {%- set _r2 = spec['$ref'] -%} + {%- set _k2 = _r2[8:] if _r2.startswith('#/$defs/') else (_r2[14:] if _r2.startswith('#/definitions/') else none) -%} + {%- if _k2 is not none and REFS.defs[_k2] is mapping -%} + {%- set spec = dict((REFS.defs[_k2] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k2 + '|' -%} + {%- endif -%} + {%- endif -%} + {%- endif -%} +{%- endif -%} +{%- if spec is mapping -%} +{{- render_markdown_schema_annotations(spec, indent, include_value_details) }} +{{- render_markdown_schema_structure(spec, indent, true) }} +{{- render_markdown_schema_extras(spec, indent) }} +{%- elif spec is not sameas true and spec is not sameas false -%} +{{- "\n" + indent + " - Value: " }}{{ render_markdown_literal(spec) }} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_markdown_parameter_schema(spec) -%} +{%- if spec is mapping and spec['$ref'] is string -%} + {%- set _r = spec['$ref'] -%} + {%- set _k = _r[8:] if _r.startswith('#/$defs/') else (_r[14:] if _r.startswith('#/definitions/') else none) -%} + {%- if _k is not none and ('|' + _k + '|') not in REFS.seen and REFS.defs[_k] is mapping -%} + {%- set spec = dict((REFS.defs[_k] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k + '|' -%} + {%- if spec['$ref'] is string -%} + {%- set _r2 = spec['$ref'] -%} + {%- set _k2 = _r2[8:] if _r2.startswith('#/$defs/') else (_r2[14:] if _r2.startswith('#/definitions/') else none) -%} + {%- if _k2 is not none and REFS.defs[_k2] is mapping -%} + {%- set spec = dict((REFS.defs[_k2] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k2 + '|' -%} + {%- endif -%} + {%- endif -%} + {%- endif -%} +{%- endif -%} +{%- if spec is mapping -%} +{{- render_markdown_metadata_annotations(spec) }} +{%- if 'items' in spec and spec['items'] is mapping -%} +{{- "\n- Items *(" + render_markdown_type(spec['items']) + ")*" }} +{{- render_markdown_schema_details(spec['items'], "", true) }} +{%- elif 'items' in spec -%} +{{ render_markdown_metadata_detail("Items", spec['items']) }} +{%- endif -%} +{%- if spec.oneOf -%} +{{- "\n- oneOf:" }} + {%- for variant in spec.oneOf -%} +{{- "\n - Variant " }}{{ loop.index }}{{- " *(" + render_markdown_type(variant) + ")*" }} +{{- render_markdown_schema_details(variant, " ", true) }} + {%- endfor -%} +{%- endif -%} +{%- if spec.anyOf -%} +{{- "\n- anyOf:" }} + {%- for variant in spec.anyOf -%} +{{- "\n - Variant " }}{{ loop.index }}{{- " *(" + render_markdown_type(variant) + ")*" }} +{{- render_markdown_schema_details(variant, " ", true) }} + {%- endfor -%} +{%- endif -%} +{%- if spec.patternProperties is mapping -%} +{{- "\n- Pattern properties:" }} + {%- for pattern, pattern_spec in spec.patternProperties | items -%} + {%- if pattern_spec is mapping -%} +{{- "\n - `" + pattern + "` *(" + render_markdown_type(pattern_spec) + ")*" }} +{{- render_markdown_schema_details(pattern_spec, " ", true) }} + {%- else -%} +{{- "\n - `" + pattern + "`: " }}{{ render_markdown_value(pattern_spec) }} + {%- endif -%} + {%- endfor -%} +{%- elif spec.patternProperties is defined -%} +{{ render_markdown_metadata_detail("Pattern properties", spec.patternProperties) }} +{%- endif -%} +{%- if spec.returns is mapping -%} +{{- "\n- Returns *(" + render_markdown_type(spec.returns) + ")*" }} +{{- render_markdown_schema_details(spec.returns, "", true) }} +{%- elif spec.returns is defined -%} +{{ render_markdown_metadata_detail("Returns", spec.returns) }} +{%- endif -%} +{{- render_markdown_metadata_extras(spec) }} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_markdown_param(name, spec, required_list, indent) -%} +{%- if spec is mapping and spec['$ref'] is string -%} + {%- set _r = spec['$ref'] -%} + {%- set _k = _r[8:] if _r.startswith('#/$defs/') else (_r[14:] if _r.startswith('#/definitions/') else none) -%} + {%- if _k is not none and ('|' + _k + '|') not in REFS.seen and REFS.defs[_k] is mapping -%} + {%- set spec = dict((REFS.defs[_k] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k + '|' -%} + {%- if spec['$ref'] is string -%} + {%- set _r2 = spec['$ref'] -%} + {%- set _k2 = _r2[8:] if _r2.startswith('#/$defs/') else (_r2[14:] if _r2.startswith('#/definitions/') else none) -%} + {%- if _k2 is not none and REFS.defs[_k2] is mapping -%} + {%- set spec = dict((REFS.defs[_k2] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k2 + '|' -%} + {%- endif -%} + {%- endif -%} + {%- endif -%} +{%- endif -%} +{{- "\n" + indent + "- `" + name + "` *(" + render_markdown_type(spec) }} +{%- if name in (required_list or []) -%}{{- ", required" }}{%- endif -%} +{{- ")*" }} +{%- if spec.description -%}{{- " - " + spec.description | replace("\n", "\n" + indent + " ") }}{%- endif -%} +{%- if spec.enum -%} +{{- "\n" + indent + " - Allowed values: " }}{{ render_allowed_values(spec.enum) }} +{%- endif -%} +{%- if spec.default is defined -%} +{{- "\n" + indent + " - Default: " }}{{ render_markdown_literal(spec.default) }} +{%- endif -%} +{{- render_markdown_schema_details(spec, indent, false) }} +{%- endmacro -%} + +{%- macro render_tools_markdown(tools_list) -%} +{{- "" }} +{%- for tool in tools_list -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set REFS.defs = fn.parameters['$defs'] if (fn.parameters is mapping and fn.parameters['$defs'] is mapping) else (fn.parameters['definitions'] if (fn.parameters is mapping and fn.parameters['definitions'] is mapping) else {}) -%} + {%- set REFS.seen = '|' -%} + {%- set fnp = namespace(p=fn.parameters) -%} + {%- if fnp.p is mapping and fnp.p['$ref'] is string -%} + {%- set _r = fnp.p['$ref'] -%} + {%- set _k = _r[8:] if _r.startswith('#/$defs/') else (_r[14:] if _r.startswith('#/definitions/') else none) -%} + {%- if _k is not none and REFS.defs[_k] is mapping -%} + {%- set fnp.p = dict((REFS.defs[_k] | items | list) + (fnp.p | items | rejectattr('0', 'equalto', '$ref') | list)) -%} + {%- set REFS.seen = REFS.seen + _k + '|' -%} + {%- endif -%} + {%- endif -%} +{{- "\n## " + fn.name }} +{%- if fn.description -%} +{{- "\n" + fn.description }} +{%- endif -%} +{{- "\n\n**Parameters**" }} +{%- if fnp.p and fnp.p.properties -%} + {%- for pname, pspec in fnp.p.properties | items -%} +{{- render_markdown_param(pname, pspec, fnp.p.required or [], "") }} + {%- endfor -%} +{%- elif fnp.p is mapping and (fnp.p.oneOf or fnp.p.anyOf or 'items' in fnp.p) -%} +{{- render_markdown_parameter_schema(fnp.p) }} +{%- else -%} +{{- "\n- None" }} +{%- endif -%} +{%- set fn_ret = fn.returns if fn.returns is defined else fn.response -%} +{%- if fn_ret is mapping -%} +{{- "\n\n**Returns**" }} +{{- "\n- Return *(" + render_markdown_type(fn_ret) + ")*" }} +{{- render_markdown_schema_details(fn_ret, "", true) }} +{%- elif fn_ret is defined -%} +{{- "\n\n**Returns**\n- " }}{{ render_markdown_value(fn_ret) }} +{%- endif -%} +{%- if not loop.last -%}{{- "\n" }}{%- endif -%} +{%- endfor -%} +{{- "\n" }} +{%- endmacro -%} + +{%- macro render_tool_presentation(tools_list, fmt) -%} +{%- if fmt == 'json' -%} +{{- render_tools_json(tools_list) }} +{%- elif RB.bad != '|' -%} +{#- some tool uses constructs the pretty renderers cannot represent (verdicts -#} +{#- computed during validate_tools): render the WHOLE toolset exactly as the -#} +{#- json presentation would, so the block stays uniform and model-familiar. -#} +{{- render_tools_json(tools_list) }} +{%- elif fmt == 'xml' -%} +{{- render_tools_xml(tools_list) }} +{%- elif fmt == 'markdown' -%} +{{- render_tools_markdown(tools_list) }} +{%- else -%} +{{- raise_exception("Unsupported tool_presentation_format: '" + fmt + "'. Supported formats: json, xml, markdown.") }} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_call_instructions(fmt) -%} +{%- if fmt == 'json' -%} +{{- "Wrap all tool calls in a single block. For each call, emit one JSON object with the function name and arguments on the same line inside tags:\n\n\n{\"name\": , \"arguments\": }\n" }} +{%- elif fmt == 'xml' -%} +{{- "Wrap all tool calls in a single block. For each call, write the function name at the start of , followed by paired and tags for each argument:\n\n\n$FUNCTION_NAME\n$PARAMETER_NAME\n$PARAMETER_VALUE\n...\n\n\n\nString and scalar parameters should be written as plain text. Array and object parameters should be written as JSON literals." }} +{%- elif fmt == 'xml_typed' -%} +{{- "Wrap all tool calls in a single block. For each call, write the function name at the start of , followed by , , and tags for each argument:\n\n\n$FUNCTION_NAME\n$PARAMETER_NAME\n$ARGUMENT_TYPE\n$PARAMETER_VALUE\n...\n\n\n\nUse the parameter type shown in the tool definition. If that type contains anyOf or oneOf, use the actual argument value type instead. String and scalar parameters should be written as plain text. Array and object parameters should be written as JSON literals." }} +{%- else -%} +{{- raise_exception("Unsupported tool_call_format: '" + fmt + "'. Supported formats: json, xml, xml_typed.") }} +{%- endif -%} +{%- endmacro -%} + +{%- macro render_system_with_tools(tools_list, system_content, presentation_fmt, call_fmt) -%} +{{- "<|ifm|im_start|>system\n# Tools\nYou may call one or more tools to assist with the user query.\n\nAvailable tools are:\n\n" }} +{{- render_tool_presentation(tools_list, presentation_fmt) }} +{{- "\n\nWhen calling tools, you MUST follow the tool-call format below:\n\n" }} +{{- render_call_instructions(call_fmt) }} +{%- if system_content -%} +{{- "\n\n" + system_content }} +{%- endif -%} +{{- "<|ifm|im_end|>" }} +{%- endmacro -%} + +{%- macro render_argument_value(value) -%} +{%- if value is string -%}{{- value -}}{%- else -%}{{- value | tojson -}}{%- endif -%} +{%- endmacro -%} + +{%- macro render_value_type(value) -%} +{%- if value is none -%}null +{%- elif value is boolean -%}boolean +{%- elif value is integer -%}integer +{%- elif value is number -%}number +{%- elif value is string -%}string +{%- elif value is mapping -%}object +{%- elif value is sequence -%}array +{%- else -%}any +{%- endif -%} +{%- endmacro -%} + +{%- macro schema_has_combinator(spec) -%} +{%- if spec.oneOf or spec.anyOf -%} +true +{%- elif spec.type is defined and spec.type is sequence and spec.type is not string and spec.type | length > 1 -%} +true +{%- elif spec.type == "array" and 'items' in spec -%} +{{- schema_has_combinator(spec['items']) -}} +{%- elif spec.properties -%} + {%- set found = namespace(value='false') -%} + {%- for child_name, child_spec in spec.properties | items -%} + {%- if schema_has_combinator(child_spec) == 'true' -%} + {%- set found.value = 'true' -%} + {%- endif -%} + {%- endfor -%} +{{- found.value -}} +{%- else -%} +false +{%- endif -%} +{%- endmacro -%} + +{%- macro render_arg_type(tools_list, tool_name, arg_name, value) -%} +{%- set found = namespace(type='any') -%} +{%- for tool in tools_list -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- if fn.name == tool_name and fn.parameters and fn.parameters.properties and arg_name in fn.parameters.properties -%} + {%- set spec = fn.parameters.properties[arg_name] -%} + {%- if spec is mapping and spec['$ref'] is string -%} + {%- set _r = spec['$ref'] -%} + {%- set _k = _r[8:] if _r.startswith('#/$defs/') else (_r[14:] if _r.startswith('#/definitions/') else none) -%} + {%- set _d = fn.parameters['$defs'] if fn.parameters['$defs'] is mapping else fn.parameters['definitions'] -%} + {%- set spec = dict((_d[_k] | items | list) + (spec | items | rejectattr('0', 'equalto', '$ref') | list)) if (_k is not none and _d is mapping and _d[_k] is mapping) else spec -%} + {%- endif -%} + {%- if schema_has_combinator(spec) == 'true' -%} + {%- set found.type = render_value_type(value) -%} + {%- else -%} + {%- set found.type = render_compact_type(spec) -%} + {%- endif -%} + {%- endif -%} +{%- endfor -%} +{{- found.type -}} +{%- endmacro -%} + +{%- macro render_tool_calls_block(tool_calls, fmt, tools_list) -%} +{{- "" }} +{%- for raw_tool_call in tool_calls -%} + {%- set tool_call = raw_tool_call.function if raw_tool_call.function else raw_tool_call -%} + {%- if tool_call.arguments is string -%} + {{- raise_exception("tool_call.arguments must be a dict, not a JSON string. Parse it before passing to the template.") -}} + {%- endif -%} + {%- if fmt == 'json' -%} +{{- "\n{\"name\": \"" + tool_call.name + "\", \"arguments\": " }}{{ tool_call.arguments | tojson }}{{- "}" }} + {%- elif fmt == 'xml' or fmt == 'xml_typed' -%} +{{- "\n" + tool_call.name + "\n" }} + {%- for key, value in tool_call.arguments | items -%} +{{- "" + key + "\n" }} +{%- if fmt == 'xml_typed' -%} +{{- "" + render_arg_type(tools_list, tool_call.name, key, value) + "\n" }} +{%- endif -%} +{{- "" }}{{ render_argument_value(value) }}{{- "\n" }} + {%- endfor -%} +{{- "" }} + {%- else -%} + {{- raise_exception("Unsupported tool_call_format: '" + fmt + "'. Supported formats: json, xml, xml_typed.") -}} + {%- endif -%} +{%- endfor -%} +{{- "\n" }} +{%- endmacro -%} + +{%- macro render_tool_response_messages(raw_content) -%} +{%- if raw_content is string -%} +{{- '<|ifm|im_start|>tool\n' + raw_content + '<|ifm|im_end|>' }} +{%- elif raw_content is sequence and raw_content is not string and raw_content is not mapping -%} + {%- if raw_content | length == 0 -%} + {{- raise_exception("tool message content list must not be empty.") -}} + {%- endif -%} +{{- '<|ifm|im_start|>tool\n' -}} + {%- for item in raw_content -%} + {%- if not loop.first -%}{{- '\n' -}}{%- endif -%} + {%- if item is string -%} +{{- item -}} + {%- elif item is mapping and item.text is string -%} +{{- item.text -}} + {%- else -%} +{{- (item | tojson) -}} + {%- endif -%} + {%- endfor -%} +{{- '<|ifm|im_end|>' -}} +{%- else -%} +{{- '<|ifm|im_start|>tool\n' }}{{ raw_content | tojson }}{{- '<|ifm|im_end|>' }} +{%- endif -%} +{%- endmacro -%} + +{%- set available_tools = tools if tools else [] -%} +{%- if (not available_tools) and messages[0].role == 'system' and messages[0].get('tools') -%} + {%- set available_tools = messages[0]['tools'] -%} +{%- endif -%} +{%- if available_tools -%} + {{- validate_tools(available_tools, tool_presentation_fmt != 'json') }} + {%- set system_content = '' -%} + {%- if messages[0].role == 'system' and messages[0].content -%} + {%- set system_content = messages[0].content -%} + {%- endif -%} + {{- render_system_with_tools(available_tools, system_content, tool_presentation_fmt, tool_call_fmt) }} +{%- else -%} + {%- if messages[0].role == 'system' -%} + {{- '<|ifm|im_start|>system\n' + messages[0].content + '<|ifm|im_end|>' }} + {%- endif -%} +{%- endif -%} + +{%- for message in messages -%} + {%- if message.content is string -%} + {%- set content = message.content -%} + {%- else -%} + {%- set content = '' -%} + {%- endif -%} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) -%} + {{- '<|ifm|im_start|>' + message.role + '\n' + content + '<|ifm|im_end|>' }} + {%- elif message.role == "assistant" -%} + {%- set thinking_content = '' -%} + {%- set think_tag = 'ifm|think' -%} + {%- if message.think is defined and message.think is string -%} + {%- set thinking_content = message.think -%} + {%- set think_tag = 'ifm|think' -%} + {%- elif message.think_fast is defined and message.think_fast is string -%} + {%- set thinking_content = message.think_fast -%} + {%- set think_tag = 'ifm|think_fast' -%} + {%- elif message.think_faster is defined and message.think_faster is string -%} + {%- set thinking_content = message.think_faster -%} + {%- set think_tag = 'ifm|think_faster' -%} + {%- elif message.reasoning_content is defined and message.reasoning_content is string -%} + {%- set thinking_content = message.reasoning_content -%} + {%- set think_tag = 'ifm|think' -%} + {%- elif message.reasoning is defined and message.reasoning is string -%} + {%- set thinking_content = message.reasoning -%} + {%- set think_tag = 'ifm|think' -%} + {%- elif message.think is not defined and message.reasoning is not defined and message.reasoning_content is not defined and message.think_fast is not defined and message.think_faster is not defined -%} + {{- raise_exception("Assistant message is missing a thinking field. Provide one of: think, reasoning, reasoning_content, think_fast, think_faster.") -}} + {%- else -%} + {{- raise_exception("Assistant thinking fields must be strings. Provide one of: think, reasoning, reasoning_content, think_fast, think_faster as a string.") -}} + {%- endif -%} + {{- '<|ifm|im_start|>' + message.role }} + {% generation %} + {%- if think_tag -%} + {%- if thinking_content -%} + {{- '<' + think_tag + '>\n' + thinking_content + '' + content }} + {%- else -%} + {{- '<' + think_tag + '>\n' + content }} + {%- endif -%} + {%- else -%} + {{- content }} + {%- endif -%} + {%- if message.tool_calls -%} + {{- render_tool_calls_block(message.tool_calls, tool_call_fmt, available_tools) }} + {%- endif -%} + {{- '<|ifm|im_end|>' -}} + {%- endgeneration -%} + {%- elif message.role == "tool" -%} + {{- render_tool_response_messages(message.content) }} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + {%- set effort = reasoning_effort | default('high') -%} + {%- if effort == 'high' -%} + {{- '<|ifm|im_start|>assistant\n\n' }} + {%- elif effort == 'medium' -%} + {{- '<|ifm|im_start|>assistant\n\n' }} + {%- elif effort == 'low' -%} + {{- '<|ifm|im_start|>assistant\n\n' }} + {%- else -%} + {{- raise_exception("Unsupported reasoning_effort: '" + effort + "'. Supported values: high, medium, low.") -}} + {%- endif -%} +{%- endif -%} diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..1a728edbc7a61fb0ae59081a2ded8a9581c0338c --- /dev/null +++ b/config.json @@ -0,0 +1,970 @@ +{ + "architectures": [ + "K2HorizonForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attention_gate_func": "softplus", + "auto_map": { + "AutoConfig": "configuration_k2_horizon.K2HorizonConfig", + "AutoModel": "modeling_k2_horizon.K2HorizonModel", + "AutoModelForCausalLM": "modeling_k2_horizon.K2HorizonForCausalLM" + }, + "bos_token_id": 0, + "decoder_sparse_step": 1, + "dtype": "bfloat16", + "eos_token_id": 1, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 6144, + "layernorm_num_groups": 2, + "max_position_embeddings": 524288, + "mlp_only_layers": [ + 0, + 1, + 2 + ], + "model_type": "k2_horizon_mova", + "moe_gate_bias": true, + "moe_intermediate_size": 768, + "mova_num_experts": 64, + "mova_num_experts_per_tok": 4, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_experts": 100, + "num_experts_per_tok": 8, + "num_hidden_layers": 48, + "num_key_value_heads": 8, + "num_shared_experts": 1, + "output_router_logits": false, + "pad_token_id": null, + "query_key_norm": false, + "rms_norm_eps": 1e-06, + "rope_head_dim": 128, + "rope_parameters": { + "rope_theta": 10000000.0, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "router_scaling_factor": 2.5, + "router_score_func": "sigmoid", + "sliding_window": null, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 250624, + "model_file": "k2_horizon_mova_mlx.py", + "quantization": { + "group_size": 64, + "bits": 4, + "mode": "affine", + "model.layers.3.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.3.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.4.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.4.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.5.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.5.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.6.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.6.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.7.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.7.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.8.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.8.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.9.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.9.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.10.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.10.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.11.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.11.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.12.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.12.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.13.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.13.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.14.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.14.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.15.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.15.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.16.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.16.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.17.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.17.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.18.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.18.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.19.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.19.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.20.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.20.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.21.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.21.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.22.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.22.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.23.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.23.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.24.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.24.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.25.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.25.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.26.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.26.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.27.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.27.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.28.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.28.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.29.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.29.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.30.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.30.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.31.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.31.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.32.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.32.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.33.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.33.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.34.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.34.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.35.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.35.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.36.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.36.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.37.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.37.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.38.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.38.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.39.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.39.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.40.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.40.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.41.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.41.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.42.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.42.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.43.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.43.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.44.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.44.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.45.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.45.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.46.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.46.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.47.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.47.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + } + }, + "quantization_config": { + "group_size": 64, + "bits": 4, + "mode": "affine", + "model.layers.3.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.3.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.4.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.4.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.5.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.5.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.6.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.6.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.7.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.7.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.8.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.8.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.9.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.9.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.10.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.10.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.11.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.11.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.12.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.12.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.13.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.13.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.14.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.14.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.15.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.15.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.16.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.16.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.17.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.17.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.18.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.18.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.19.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.19.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.20.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.20.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.21.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.21.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.22.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.22.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.23.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.23.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.24.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.24.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.25.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.25.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.26.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.26.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.27.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.27.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.28.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.28.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.29.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.29.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.30.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.30.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.31.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.31.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.32.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.32.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.33.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.33.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.34.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.34.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.35.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.35.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.36.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.36.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.37.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.37.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.38.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.38.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.39.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.39.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.40.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.40.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.41.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.41.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.42.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.42.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.43.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.43.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.44.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.44.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.45.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.45.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.46.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.46.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.47.mlp.gate": { + "group_size": 64, + "bits": 8, + "mode": "affine" + }, + "model.layers.47.self_attn.v_router": { + "group_size": 64, + "bits": 8, + "mode": "affine" + } + } +} diff --git a/generation_config.json b/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..1a96c073d351630d0cf6738b9825a9962890d0ea --- /dev/null +++ b/generation_config.json @@ -0,0 +1,7 @@ +{ + "bos_token_id": 0, + "eos_token_id": [ + 1, + 250019 + ] +} diff --git a/k2_horizon_mova_mlx.py b/k2_horizon_mova_mlx.py new file mode 100644 index 0000000000000000000000000000000000000000..374e7e7c65d22c7a4880f0e31156b19ab7e85f37 --- /dev/null +++ b/k2_horizon_mova_mlx.py @@ -0,0 +1,269 @@ +"""MLX-LM adapter for IFM K2-Horizon-MoVA-36B-A4B. + +The model is not a standard Llama/Mixtral checkpoint: sparse decoder layers +use both sigmoid-routed feed-forward experts and routed value experts in +attention (MoVA). This module mirrors the upstream K2 Horizon implementation +without substituting its grouped RMSNorm, routing rules, or attention gate. +""" + +from dataclasses import dataclass +import math +from typing import Any, Dict, List, Optional, Union + +import mlx.core as mx +import mlx.nn as nn + +from mlx_lm.models.activations import swiglu +from mlx_lm.models.base import BaseModelArgs, create_attention_mask, scaled_dot_product_attention +from mlx_lm.models.cache import KVCache +from mlx_lm.models.switch_layers import SwitchGLU, SwitchLinear + + +@dataclass +class ModelArgs(BaseModelArgs): + model_type: str + hidden_size: int + num_hidden_layers: int + intermediate_size: int + moe_intermediate_size: int + num_attention_heads: int + num_key_value_heads: int + num_experts: int + num_experts_per_tok: int + mova_num_experts: int + mova_num_experts_per_tok: int + num_shared_experts: int + decoder_sparse_step: int + mlp_only_layers: List[int] + rms_norm_eps: float + vocab_size: int + head_dim: int + max_position_embeddings: int + norm_topk_prob: bool + router_score_func: str + router_scaling_factor: float + tie_word_embeddings: bool + layernorm_num_groups: int = 2 + rope_theta: float = 10_000_000.0 + rope_head_dim: Optional[int] = None + attention_bias: bool = False + moe_gate_bias: bool = True + attention_gate_func: Optional[str] = None + rope_scaling: Optional[Dict[str, Union[float, str]]] = None + + +class GroupRMSNorm(nn.Module): + def __init__(self, dims: int, eps: float, groups: int): + super().__init__() + if dims % groups: + raise ValueError(f"hidden size {dims} is not divisible by {groups} groups") + self.weight = mx.ones((dims,)) + self.groups = groups + self.eps = eps + + def __call__(self, x: mx.array) -> mx.array: + x = mx.unflatten(x, axis=-1, shape=(self.groups, -1)) + x = mx.fast.rms_norm(x, weight=None, eps=self.eps) + return self.weight * mx.flatten(x, -2) + + +def _router(x: mx.array, gate: nn.Linear, top_k: int, *, normalize: bool, scale: float): + """K2 router: bias affects top-k selection but not sigmoid scores.""" + # Calling the module (rather than reading ``weight`` directly) also works + # after it has become a QuantizedLinear. K2 applies the router bias only + # when choosing experts, so remove it from the score logits first. + logits = gate(x) + if "bias" in gate: + logits = logits - gate.bias + scores = mx.sigmoid(logits.astype(mx.float32)) + choice_scores = scores + gate.bias.astype(scores.dtype) if "bias" in gate else scores + indices = mx.stop_gradient(mx.argpartition(choice_scores, kth=-top_k, axis=-1)[..., -top_k:]) + weights = mx.take_along_axis(scores, indices, axis=-1) + if normalize: + weights = weights / mx.sum(weights, axis=-1, keepdims=True) + return weights.astype(x.dtype) * scale, indices + + +def _attention_gate(x: mx.array, projection: Optional[nn.Linear], func: Optional[str], heads: int, head_dim: int): + if projection is None: + return None + gate = projection(x).reshape(*x.shape[:-1], heads, head_dim) + if func == "silu": + return nn.silu(gate) + if func == "softplus": + beta = math.log(2.0) + return mx.log1p(mx.exp(gate * beta)) / beta + raise ValueError(f"unsupported attention gate: {func}") + + +class DenseAttention(nn.Module): + def __init__(self, args: ModelArgs): + super().__init__() + dim = args.hidden_size + self.n_heads = args.num_attention_heads + self.n_kv_heads = args.num_key_value_heads + self.head_dim = args.head_dim + self.scale = self.head_dim ** -0.5 + self.q_proj = nn.Linear(dim, self.n_heads * self.head_dim, bias=args.attention_bias) + self.k_proj = nn.Linear(dim, self.n_kv_heads * self.head_dim, bias=args.attention_bias) + self.v_proj = nn.Linear(dim, self.n_kv_heads * self.head_dim, bias=args.attention_bias) + self.o_proj = nn.Linear(self.n_heads * self.head_dim, dim, bias=args.attention_bias) + self.gate_proj = ( + nn.Linear(dim, self.n_heads * self.head_dim, bias=False) + if args.attention_gate_func is not None else None + ) + self.gate_func = args.attention_gate_func + self.rope = nn.RoPE(self.head_dim, traditional=False, base=args.rope_theta) + + def __call__(self, x: mx.array, mask=None, cache=None) -> mx.array: + b, length, _ = x.shape + q = self.q_proj(x).reshape(b, length, self.n_heads, self.head_dim).transpose(0, 2, 1, 3) + k = self.k_proj(x).reshape(b, length, self.n_kv_heads, self.head_dim).transpose(0, 2, 1, 3) + v = self.v_proj(x).reshape(b, length, self.n_kv_heads, self.head_dim).transpose(0, 2, 1, 3) + offset = cache.offset if cache is not None else 0 + q, k = self.rope(q, offset=offset), self.rope(k, offset=offset) + if cache is not None: + k, v = cache.update_and_fetch(k, v) + out = scaled_dot_product_attention(q, k, v, cache=cache, scale=self.scale, mask=mask) + gate = _attention_gate(x, self.gate_proj, self.gate_func, self.n_heads, self.head_dim) + if gate is not None: + out = out * gate.transpose(0, 2, 1, 3) + return self.o_proj(out.transpose(0, 2, 1, 3).reshape(b, length, -1)) + + +class MoVAAttention(nn.Module): + def __init__(self, args: ModelArgs): + super().__init__() + dim = args.hidden_size + self.n_heads = args.num_attention_heads + self.n_kv_heads = args.num_key_value_heads + self.head_dim = args.head_dim + self.scale = self.head_dim ** -0.5 + self.top_k = args.mova_num_experts_per_tok + self.router_scale = args.router_scaling_factor + self.q_proj = nn.Linear(dim, self.n_heads * self.head_dim, bias=args.attention_bias) + self.k_proj = nn.Linear(dim, self.n_kv_heads * self.head_dim, bias=args.attention_bias) + self.o_proj = nn.Linear(self.n_heads * self.head_dim, dim, bias=args.attention_bias) + self.v_router = nn.Linear(dim, args.mova_num_experts, bias=args.moe_gate_bias) + self.v_experts = SwitchLinear(dim, self.n_kv_heads * self.head_dim, args.mova_num_experts, bias=False) + self.gate_proj = ( + nn.Linear(dim, self.n_heads * self.head_dim, bias=False) + if args.attention_gate_func is not None else None + ) + self.gate_func = args.attention_gate_func + self.rope = nn.RoPE(self.head_dim, traditional=False, base=args.rope_theta) + + def __call__(self, x: mx.array, mask=None, cache=None) -> mx.array: + b, length, _ = x.shape + flat = x.reshape(-1, x.shape[-1]) + weights, indices = _router(flat, self.v_router, self.top_k, normalize=True, scale=self.router_scale) + routed = self.v_experts(mx.expand_dims(flat, (-2, -3)), indices).squeeze(-2) + values = (nn.silu(routed) * weights[..., None]).sum(axis=-2) + v = values.reshape(b, length, self.n_kv_heads, self.head_dim).transpose(0, 2, 1, 3) + q = self.q_proj(x).reshape(b, length, self.n_heads, self.head_dim).transpose(0, 2, 1, 3) + k = self.k_proj(x).reshape(b, length, self.n_kv_heads, self.head_dim).transpose(0, 2, 1, 3) + offset = cache.offset if cache is not None else 0 + q, k = self.rope(q, offset=offset), self.rope(k, offset=offset) + if cache is not None: + k, v = cache.update_and_fetch(k, v) + out = scaled_dot_product_attention(q, k, v, cache=cache, scale=self.scale, mask=mask) + gate = _attention_gate(x, self.gate_proj, self.gate_func, self.n_heads, self.head_dim) + if gate is not None: + out = out * gate.transpose(0, 2, 1, 3) + return self.o_proj(out.transpose(0, 2, 1, 3).reshape(b, length, -1)) + + +class MLP(nn.Module): + def __init__(self, dim: int, hidden_dim: int): + super().__init__() + self.gate_proj = nn.Linear(dim, hidden_dim, bias=False) + self.up_proj = nn.Linear(dim, hidden_dim, bias=False) + self.down_proj = nn.Linear(hidden_dim, dim, bias=False) + + def __call__(self, x: mx.array) -> mx.array: + return self.down_proj(swiglu(self.gate_proj(x), self.up_proj(x))) + + +class SparseMoE(nn.Module): + def __init__(self, args: ModelArgs): + super().__init__() + self.top_k = args.num_experts_per_tok + self.router_scale = args.router_scaling_factor + self.gate = nn.Linear(args.hidden_size, args.num_experts, bias=args.moe_gate_bias) + self.switch_mlp = SwitchGLU(args.hidden_size, args.moe_intermediate_size, args.num_experts, bias=False) + self.shared_experts = MLP(args.hidden_size, args.moe_intermediate_size * args.num_shared_experts) + + def __call__(self, x: mx.array) -> mx.array: + weights, indices = _router(x, self.gate, self.top_k, normalize=True, scale=self.router_scale) + y = self.switch_mlp(x, indices) + y = (y * weights[..., None]).sum(axis=-2).astype(x.dtype) + return y + self.shared_experts(x) + + +class DecoderLayer(nn.Module): + def __init__(self, args: ModelArgs, index: int): + super().__init__() + sparse = index not in args.mlp_only_layers and args.num_experts > 0 and (index + 1) % args.decoder_sparse_step == 0 + self.self_attn = MoVAAttention(args) if sparse and args.mova_num_experts > 0 else DenseAttention(args) + self.mlp = SparseMoE(args) if sparse else MLP(args.hidden_size, args.intermediate_size) + self.input_layernorm = GroupRMSNorm(args.hidden_size, args.rms_norm_eps, args.layernorm_num_groups) + self.post_attention_layernorm = GroupRMSNorm(args.hidden_size, args.rms_norm_eps, args.layernorm_num_groups) + + def __call__(self, x: mx.array, mask=None, cache=None) -> mx.array: + h = x + self.self_attn(self.input_layernorm(x), mask, cache) + return h + self.mlp(self.post_attention_layernorm(h)) + + +class K2HorizonModel(nn.Module): + def __init__(self, args: ModelArgs): + super().__init__() + self.embed_tokens = nn.Embedding(args.vocab_size, args.hidden_size) + self.layers = [DecoderLayer(args, i) for i in range(args.num_hidden_layers)] + self.norm = GroupRMSNorm(args.hidden_size, args.rms_norm_eps, args.layernorm_num_groups) + + def __call__(self, inputs: mx.array, cache=None, input_embeddings=None) -> mx.array: + h = self.embed_tokens(inputs) if input_embeddings is None else input_embeddings + if cache is None: + cache = [None] * len(self.layers) + mask = create_attention_mask(h, cache[0]) + for layer, state in zip(self.layers, cache): + h = layer(h, mask, state) + return self.norm(h) + + +class Model(nn.Module): + def __init__(self, args: ModelArgs): + super().__init__() + self.args = args + self.model_type = args.model_type + self.model = K2HorizonModel(args) + self.lm_head = nn.Linear(args.hidden_size, args.vocab_size, bias=False) + + def __call__(self, inputs: mx.array, cache=None, input_embeddings=None) -> mx.array: + return self.lm_head(self.model(inputs, cache, input_embeddings)) + + def sanitize(self, weights): + weights.pop("model.rotary_emb.inv_freq", None) + for layer in range(self.args.num_hidden_layers): + prefix = f"model.layers.{layer}" + if f"{prefix}.mlp.experts.0.up_proj.weight" in weights: + for name in ("up_proj", "down_proj", "gate_proj"): + expert_weights = [weights.pop(f"{prefix}.mlp.experts.{expert}.{name}.weight") for expert in range(self.args.num_experts)] + weights[f"{prefix}.mlp.switch_mlp.{name}.weight"] = mx.stack(expert_weights) + if f"{prefix}.self_attn.v_experts.0.weight" in weights: + expert_weights = [weights.pop(f"{prefix}.self_attn.v_experts.{expert}.weight") for expert in range(self.args.mova_num_experts)] + weights[f"{prefix}.self_attn.v_experts.weight"] = mx.stack(expert_weights) + return weights + + @property + def quant_predicate(self): + def predicate(path, _): + # Routing weights are numerically sensitive and remain at 8-bit. + if path.endswith("mlp.gate") or path.endswith("self_attn.v_router"): + return {"group_size": 64, "bits": 8} + return True + return predicate + + @property + def layers(self): + return self.model.layers diff --git a/model-00001-of-00048.safetensors b/model-00001-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2a74252ca96e09b670121262bbe75ae427a1d646 --- /dev/null +++ b/model-00001-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:31e5235971c9831d85f84ffa50bf5fe4bebc410c99aac3026073b231c64fdeee +size 47199035 diff --git a/model-00002-of-00048.safetensors b/model-00002-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1005ae36b213687744f4ac27ec3ebcb6b219241a --- /dev/null +++ b/model-00002-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d08fb67b1a30b7150e73eecdce19a76d0fd273a735bd318d32242070ca71612 +size 47199021 diff --git a/model-00003-of-00048.safetensors b/model-00003-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..eed6c60e3409a2ddc20742dade2faa253919be2e --- /dev/null +++ b/model-00003-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1bd9d907b1104667c2f5572d451823c9fed5229d7b7b411b19b1ca56ed71863a +size 47199029 diff --git a/model-00004-of-00048.safetensors b/model-00004-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8840080fe808f2159582196a3dd3c79d79f6458f --- /dev/null +++ b/model-00004-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fcf17a50a793a7d6c75efa7890cd45177e1dff6dcf87934075ca236abd5d82e5 +size 449096574 diff --git a/model-00005-of-00048.safetensors b/model-00005-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..bea012c7156afe2e6fdac8870023a09019a376b1 --- /dev/null +++ b/model-00005-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c60a0c455f5d866944a8ab1d1b6b31ec11f0e83b3333365f9cf6ffbbf2541684 +size 449096556 diff --git a/model-00006-of-00048.safetensors b/model-00006-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..af672f9d71a3763403442afbb138f831c851c600 --- /dev/null +++ b/model-00006-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51401f4e04332968247f3d1525359647bb5f810e457f577772a5c7d8510bceef +size 449096572 diff --git a/model-00007-of-00048.safetensors b/model-00007-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f25794ed355bd932fa61ba60c5f73e5f1780ecd5 --- /dev/null +++ b/model-00007-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50a7ea11975014ee47cdbbc1a154e2bb8167ea0303c943d60c6a91804efdc737 +size 449096572 diff --git a/model-00008-of-00048.safetensors b/model-00008-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a514190520eebb7733f4a63be168e01dc0af6210 --- /dev/null +++ b/model-00008-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9558e7e8f36cc59073dddb1e9493b04b61dafee961f0bf7ac37d9ba171558dba +size 449096572 diff --git a/model-00009-of-00048.safetensors b/model-00009-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..16287d6fff9cabafc63128bc25d15cc16ff978f0 --- /dev/null +++ b/model-00009-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6636e298689e80a83668d798170e5a9f2988cbc97684f220319001be98b3bec1 +size 449096572 diff --git a/model-00010-of-00048.safetensors b/model-00010-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..c838c0fcbb8f0cb2ba16e4c3d6583d60a4f383e7 --- /dev/null +++ b/model-00010-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc7d03e8f8841b5eb7bc8884a2e244c09aea5f29ee136837614bafdf939cb042 +size 449096574 diff --git a/model-00011-of-00048.safetensors b/model-00011-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8ea30880cbc46a90ecdae57dff55c463089edbb9 --- /dev/null +++ b/model-00011-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3308837f6d5eed5bc980446b69f88df0444d9041e4cfc86e495a9a6b776dc733 +size 449096615 diff --git a/model-00012-of-00048.safetensors b/model-00012-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9004ef51038b11fbd9722ab95150b4c3c0114c12 --- /dev/null +++ b/model-00012-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30c4d8f41d8586da70e3d6a2f193cea93b7b54ee5eff8a647eebb42c223e6ee9 +size 449096617 diff --git a/model-00013-of-00048.safetensors b/model-00013-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..029fff8a399c0578a4db874c8c192c4bc19d38a9 --- /dev/null +++ b/model-00013-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53302fc7cf9f635ad3b536df999f1a30a2ac9b6bbd34981473d3e07a8924fbbf +size 449096611 diff --git a/model-00014-of-00048.safetensors b/model-00014-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a67f37d2e1dd06e6f496f7e387dc10ef211e3c6d --- /dev/null +++ b/model-00014-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33a6bd51e600e5f767d7cc7b898bf958e8fb14573fa76f204e9b3a461d3d1da1 +size 449096619 diff --git a/model-00015-of-00048.safetensors b/model-00015-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b39d449064ad49503adde922939bdb8149d46ba6 --- /dev/null +++ b/model-00015-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7140efd36587a70c9984adc0aa00523951d0a3b9feda24134faa4bccc93a8457 +size 449096617 diff --git a/model-00016-of-00048.safetensors b/model-00016-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..848b56a86ccb3c497aab3e07d53eeb53ffaa03da --- /dev/null +++ b/model-00016-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9371d749bc50601fccf32d6ed497fc9e3c6cf8f228cabbe9ed847a4c2bd69840 +size 449096615 diff --git a/model-00017-of-00048.safetensors b/model-00017-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..2475f441b9ccd83114bf9d9e3f0a3f8a1714859e --- /dev/null +++ b/model-00017-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:027c61bb1e5ea9582bc6fb9349b9e5f1633f47db1ee4635d1be6600d95b39c85 +size 449096575 diff --git a/model-00018-of-00048.safetensors b/model-00018-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..74c7f4f9ea2c7685ff9689dc15a52fd026081124 --- /dev/null +++ b/model-00018-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6dbe8d96f1c83389dc96a4a2fc7722924c48b63e3959cd3aaa771a7b1ba2e54 +size 449096611 diff --git a/model-00019-of-00048.safetensors b/model-00019-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..114ec536ced7142d7e57db8ac483337945cd3704 --- /dev/null +++ b/model-00019-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:455100cce3c5be76660c62d368e44491c963e7bbb10062cf85cfacf886753751 +size 449096615 diff --git a/model-00020-of-00048.safetensors b/model-00020-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f558aab209b2fb64862b1ea3ac02b08d30753691 --- /dev/null +++ b/model-00020-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e72327b996a2fad2198d21b49afb2850c3644ec372847525156604029d2a183a +size 449096615 diff --git a/model-00021-of-00048.safetensors b/model-00021-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3d3fd3a8ba9f2d7ef9007e87eee87b20a5b98a0f --- /dev/null +++ b/model-00021-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b8c5f6ea0b6207283dc02df0b20b23f462fd4a68d8716a2e74a1cb69786c452 +size 449096619 diff --git a/model-00022-of-00048.safetensors b/model-00022-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..7b2da8493fc8a98aa8e7ef671c46cea5d80448ed --- /dev/null +++ b/model-00022-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4ec0318b4e56e797be38ae57b5c72b65f9ab681179f2368467493aa556a74ca +size 449096615 diff --git a/model-00023-of-00048.safetensors b/model-00023-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..258aaee9e3bcb24c046e1eef6fc397e44e7882ee --- /dev/null +++ b/model-00023-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b8c0abcaae152db6d66ae069c29d81476b9786c9b9d485ebf3705fbe39ef7df +size 449096615 diff --git a/model-00024-of-00048.safetensors b/model-00024-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..39fe11bbcc2afa7b95d346d4daf3681a0e048d2b --- /dev/null +++ b/model-00024-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db2f9bfb742391e8ed2d53bb83ba78ab0b5225580ccb6018b8c0d8e5bd4fb0e0 +size 449096613 diff --git a/model-00025-of-00048.safetensors b/model-00025-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ac55930cc5ca8c668c7c62673eb0eb87a1cc688a --- /dev/null +++ b/model-00025-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:70f63ca25cc527417a1cad79cf58bcb5616f35f97f4027d0fddeb9ad5196438b +size 449096611 diff --git a/model-00026-of-00048.safetensors b/model-00026-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..e43fe1869c90e72a57d83520bc4fa48664c74096 --- /dev/null +++ b/model-00026-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89b205e4659789bfd58bf49004a76b4fa99d9bbdd1fcae5ec861ea4472677627 +size 449096613 diff --git a/model-00027-of-00048.safetensors b/model-00027-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..832fa852e50534c9019fa28e5ab0e47a66ef2918 --- /dev/null +++ b/model-00027-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:209af642e45016b37be905086e163b4ca111dd33cf48897317ed7b243e8c970e +size 449096615 diff --git a/model-00028-of-00048.safetensors b/model-00028-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..5e2e2ccbacfc60004b9d6b56a3ebf35003c1aa16 --- /dev/null +++ b/model-00028-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd11b46f53a8c83da2f2311309065e33634235001a236e1c717a65f201029af7 +size 449096619 diff --git a/model-00029-of-00048.safetensors b/model-00029-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..27ac3574b8d2b91045ee81bb863085e946a1b5ae --- /dev/null +++ b/model-00029-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:461c72b2368caa565d95613d9f707f18621a7a2b76b5ddbedfbdbe37589d8c80 +size 449096619 diff --git a/model-00030-of-00048.safetensors b/model-00030-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3e3ff8908e00d93d7f17d68964c1b6b45886aecd --- /dev/null +++ b/model-00030-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a2ed651142e80d299978a6c1caabd4e23fb5650e7ef3e11bfec967d921ffa97 +size 449096617 diff --git a/model-00031-of-00048.safetensors b/model-00031-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..be9c8dc8986f1d7378aa128c79ab06c19c2d3422 --- /dev/null +++ b/model-00031-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6dc06c4d8b90ceb7461ed5bddd9134a08c05695972c00e66d38599a2efe7b894 +size 449096613 diff --git a/model-00032-of-00048.safetensors b/model-00032-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f978c3dc8150421564e7071a21747ebc66c8e2c4 --- /dev/null +++ b/model-00032-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c25408fc0a757c47128efbfed6cfd1faeca4641c79931461e0b0385c83eec80e +size 449096615 diff --git a/model-00033-of-00048.safetensors b/model-00033-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d0accf54dab64bd983f1b3d603ea80d9982f0ca4 --- /dev/null +++ b/model-00033-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e35e950738d0e9fa5e272d87b91d6368b0c90a73b1aa378c1144cc28fabdd5a +size 449096615 diff --git a/model-00034-of-00048.safetensors b/model-00034-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..56d49643be20316f3e5d5c7fdf0d77091510c100 --- /dev/null +++ b/model-00034-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce9adbb1346a4bb7049e96ceff3ca7b68c2d8550ffbabee69b5bf45c5e6f0564 +size 449096615 diff --git a/model-00035-of-00048.safetensors b/model-00035-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6d1ca532214ba556ec3a95dc1a013c74359aeab5 --- /dev/null +++ b/model-00035-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2c64688e850c215c39759a030ab03acad66e6953b82e75be7ed86563893d5ec +size 449096615 diff --git a/model-00036-of-00048.safetensors b/model-00036-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..388fb7e921498c3828f2ad13dcfe4653424f0de3 --- /dev/null +++ b/model-00036-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6b182ec03b2aee2fe6a8abda92a0da562839299833a1228d8ccf0c05e23125cc +size 449096615 diff --git a/model-00037-of-00048.safetensors b/model-00037-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1f44338b545ce66df8a6a5871dbf7bf8f04b60d2 --- /dev/null +++ b/model-00037-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e2721a914cd08404fb97f3cc5a1ac2b637c649e517db2bbb4e2b969dc5eee8f +size 449096617 diff --git a/model-00038-of-00048.safetensors b/model-00038-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..410a80e4b1fa83eecee564849566d7ffbe3d2de5 --- /dev/null +++ b/model-00038-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:350aa2c893d468fdb7bd38f5ae026f230198654f48bd57e14b4b00bce5ad94c8 +size 449096619 diff --git a/model-00039-of-00048.safetensors b/model-00039-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..528193292a0e3066e4955b6196ffa0e3f101b51f --- /dev/null +++ b/model-00039-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9491cd273404aad9d152183969870833ac785016938f76abcf40ffca02609a3 +size 449096611 diff --git a/model-00040-of-00048.safetensors b/model-00040-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..9f2996e1875cc644664dcd87fc6b2656eb2063e0 --- /dev/null +++ b/model-00040-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0692877de523630bb33b782d71f690abce2f8af23b3357ec817b3bf2a4d5de2 +size 449096619 diff --git a/model-00041-of-00048.safetensors b/model-00041-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..f04dd83e98b7cf77666f558a79def063084c30ae --- /dev/null +++ b/model-00041-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6502976c7222801e6839aef84c666b4766aecc8fb42680804c1ee7b5fccfeb5 +size 449096617 diff --git a/model-00042-of-00048.safetensors b/model-00042-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6b1372e1549da2150744fc36117859a50884dddc --- /dev/null +++ b/model-00042-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:61618b4716835ec1e3405163a9def07e70745dcb1bd1583c783b655b1b35d732 +size 449096615 diff --git a/model-00043-of-00048.safetensors b/model-00043-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..b242a29ed57b6c1cd42f1dd4053147c7d2bf1b4d --- /dev/null +++ b/model-00043-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:755485d662cb6e52916f1e00e27e0548bb46ea1363945f296224ea1ede50681c +size 449096615 diff --git a/model-00044-of-00048.safetensors b/model-00044-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6b9d64f245997e89bcc9c9b9b8df650d89fa6f7d --- /dev/null +++ b/model-00044-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f93a315a0732690c3dbbe80dfd32c23a1eb2d930c075b2f4ae7dae11134760ec +size 449096617 diff --git a/model-00045-of-00048.safetensors b/model-00045-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ab0e9f4b0c1592cf92b0f32b1ded4d98b7e62bc7 --- /dev/null +++ b/model-00045-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b53e9662206809c13ea095de8d324a3acfe53b35f9cccc4baa8b22c39ffdff26 +size 449096619 diff --git a/model-00046-of-00048.safetensors b/model-00046-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3b64c197c68da59de420ac5568831d463411caa3 --- /dev/null +++ b/model-00046-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:346ffb61c16963d6e15fe685e848155bf7f92f5e23a97cb15b5e8f62713b48d5 +size 449096617 diff --git a/model-00047-of-00048.safetensors b/model-00047-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..1b40b573f0cf659c1d9afa61c3307a0b978d8f06 --- /dev/null +++ b/model-00047-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d89a0b584c6ba903950544f62bae40f5d503e734e574d892402fa6ed34783f73 +size 449096615 diff --git a/model-00048-of-00048.safetensors b/model-00048-of-00048.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cb621def46e4d07082cb39a69fbf06e2c8e21a74 --- /dev/null +++ b/model-00048-of-00048.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc24affec2ce026fe5ce6036812689866b3efc17716d23ec602e67ae4111f63 +size 1170899542 diff --git a/model.safetensors.index.json b/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..f4bd9a5d132bfaa8fcb6972505695eca65e1d7e0 --- /dev/null +++ b/model.safetensors.index.json @@ -0,0 +1,2030 @@ +{ + "metadata": { + "dtype": "bfloat16", + "format": "mlx", + "source": "/lustrefs/users/bbqbyte/workspace/checkpoints/xllm/k2mova-36B_mid5_decay_50B_jais250k_bsz20M_seq512k_lr4e-5_linear_wd0.06_rope128_dot_te/checkpoints/checkpoint_0002500", + "total_size": 74889584040 + }, + "weight_map": { + "lm_head.biases": "model-00048-of-00048.safetensors", + "lm_head.scales": "model-00048-of-00048.safetensors", + "lm_head.weight": "model-00048-of-00048.safetensors", + "model.embed_tokens.biases": "model-00048-of-00048.safetensors", + "model.embed_tokens.scales": "model-00048-of-00048.safetensors", + "model.embed_tokens.weight": "model-00048-of-00048.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00048.safetensors", + "model.layers.0.mlp.down_proj.biases": "model-00001-of-00048.safetensors", + "model.layers.0.mlp.down_proj.scales": "model-00001-of-00048.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00048.safetensors", + "model.layers.0.mlp.gate_proj.biases": "model-00001-of-00048.safetensors", + "model.layers.0.mlp.gate_proj.scales": "model-00001-of-00048.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00048.safetensors", + "model.layers.0.mlp.up_proj.biases": "model-00001-of-00048.safetensors", + "model.layers.0.mlp.up_proj.scales": "model-00001-of-00048.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00048.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.gate_proj.biases": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.gate_proj.scales": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.gate_proj.weight": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.k_proj.biases": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.k_proj.scales": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.o_proj.biases": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.o_proj.scales": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.q_proj.biases": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.q_proj.scales": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.v_proj.biases": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.v_proj.scales": "model-00001-of-00048.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00048.safetensors", + "model.layers.1.input_layernorm.weight": "model-00002-of-00048.safetensors", + "model.layers.1.mlp.down_proj.biases": "model-00002-of-00048.safetensors", + "model.layers.1.mlp.down_proj.scales": "model-00002-of-00048.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00002-of-00048.safetensors", + "model.layers.1.mlp.gate_proj.biases": "model-00002-of-00048.safetensors", + "model.layers.1.mlp.gate_proj.scales": "model-00002-of-00048.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00002-of-00048.safetensors", + "model.layers.1.mlp.up_proj.biases": "model-00002-of-00048.safetensors", + "model.layers.1.mlp.up_proj.scales": "model-00002-of-00048.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00002-of-00048.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.gate_proj.biases": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.gate_proj.scales": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.gate_proj.weight": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.k_proj.biases": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.k_proj.scales": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.o_proj.biases": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.o_proj.scales": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.q_proj.biases": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.q_proj.scales": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.v_proj.biases": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.v_proj.scales": "model-00002-of-00048.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00002-of-00048.safetensors", + "model.layers.10.input_layernorm.weight": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.gate.bias": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.gate.biases": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.gate.scales": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.gate.weight": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.shared_experts.down_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.shared_experts.down_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.shared_experts.down_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.shared_experts.gate_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.shared_experts.gate_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.shared_experts.gate_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.shared_experts.up_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.shared_experts.up_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.shared_experts.up_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.switch_mlp.down_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.switch_mlp.down_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.switch_mlp.down_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.switch_mlp.gate_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.switch_mlp.gate_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.switch_mlp.gate_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.switch_mlp.up_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.switch_mlp.up_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.mlp.switch_mlp.up_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.gate_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.gate_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.gate_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.k_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.k_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.o_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.o_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.q_proj.biases": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.q_proj.scales": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.v_experts.biases": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.v_experts.scales": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.v_experts.weight": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.v_router.bias": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.v_router.biases": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.v_router.scales": "model-00011-of-00048.safetensors", + "model.layers.10.self_attn.v_router.weight": "model-00011-of-00048.safetensors", + "model.layers.11.input_layernorm.weight": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.gate.bias": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.gate.biases": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.gate.scales": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.gate.weight": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.shared_experts.down_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.shared_experts.down_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.shared_experts.down_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.shared_experts.gate_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.shared_experts.gate_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.shared_experts.gate_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.shared_experts.up_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.shared_experts.up_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.shared_experts.up_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.switch_mlp.down_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.switch_mlp.down_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.switch_mlp.down_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.switch_mlp.gate_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.switch_mlp.gate_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.switch_mlp.gate_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.switch_mlp.up_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.switch_mlp.up_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.mlp.switch_mlp.up_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.gate_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.gate_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.gate_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.k_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.k_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.o_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.o_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.q_proj.biases": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.q_proj.scales": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.v_experts.biases": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.v_experts.scales": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.v_experts.weight": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.v_router.bias": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.v_router.biases": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.v_router.scales": "model-00012-of-00048.safetensors", + "model.layers.11.self_attn.v_router.weight": "model-00012-of-00048.safetensors", + "model.layers.12.input_layernorm.weight": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.gate.bias": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.gate.biases": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.gate.scales": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.gate.weight": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.shared_experts.down_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.shared_experts.down_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.shared_experts.down_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.shared_experts.gate_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.shared_experts.gate_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.shared_experts.gate_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.shared_experts.up_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.shared_experts.up_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.shared_experts.up_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.switch_mlp.down_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.switch_mlp.down_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.switch_mlp.down_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.switch_mlp.gate_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.switch_mlp.gate_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.switch_mlp.gate_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.switch_mlp.up_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.switch_mlp.up_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.mlp.switch_mlp.up_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.gate_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.gate_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.gate_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.k_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.k_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.o_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.o_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.q_proj.biases": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.q_proj.scales": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.v_experts.biases": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.v_experts.scales": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.v_experts.weight": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.v_router.bias": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.v_router.biases": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.v_router.scales": "model-00013-of-00048.safetensors", + "model.layers.12.self_attn.v_router.weight": "model-00013-of-00048.safetensors", + "model.layers.13.input_layernorm.weight": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.gate.bias": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.gate.biases": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.gate.scales": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.gate.weight": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.shared_experts.down_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.shared_experts.down_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.shared_experts.down_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.shared_experts.gate_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.shared_experts.gate_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.shared_experts.gate_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.shared_experts.up_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.shared_experts.up_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.shared_experts.up_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.switch_mlp.down_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.switch_mlp.down_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.switch_mlp.down_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.switch_mlp.gate_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.switch_mlp.gate_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.switch_mlp.gate_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.switch_mlp.up_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.switch_mlp.up_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.mlp.switch_mlp.up_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.gate_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.gate_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.gate_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.k_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.k_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.o_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.o_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.q_proj.biases": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.q_proj.scales": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.v_experts.biases": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.v_experts.scales": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.v_experts.weight": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.v_router.bias": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.v_router.biases": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.v_router.scales": "model-00014-of-00048.safetensors", + "model.layers.13.self_attn.v_router.weight": "model-00014-of-00048.safetensors", + "model.layers.14.input_layernorm.weight": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.gate.bias": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.gate.biases": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.gate.scales": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.gate.weight": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.shared_experts.down_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.shared_experts.down_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.shared_experts.down_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.shared_experts.gate_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.shared_experts.gate_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.shared_experts.gate_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.shared_experts.up_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.shared_experts.up_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.shared_experts.up_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.switch_mlp.down_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.switch_mlp.down_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.switch_mlp.down_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.switch_mlp.gate_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.switch_mlp.gate_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.switch_mlp.gate_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.switch_mlp.up_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.switch_mlp.up_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.mlp.switch_mlp.up_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.gate_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.gate_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.gate_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.k_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.k_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.o_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.o_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.q_proj.biases": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.q_proj.scales": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.v_experts.biases": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.v_experts.scales": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.v_experts.weight": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.v_router.bias": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.v_router.biases": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.v_router.scales": "model-00015-of-00048.safetensors", + "model.layers.14.self_attn.v_router.weight": "model-00015-of-00048.safetensors", + "model.layers.15.input_layernorm.weight": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.gate.bias": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.gate.biases": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.gate.scales": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.gate.weight": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.shared_experts.down_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.shared_experts.down_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.shared_experts.down_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.shared_experts.gate_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.shared_experts.gate_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.shared_experts.gate_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.shared_experts.up_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.shared_experts.up_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.shared_experts.up_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.switch_mlp.down_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.switch_mlp.down_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.switch_mlp.down_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.switch_mlp.gate_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.switch_mlp.gate_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.switch_mlp.gate_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.switch_mlp.up_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.switch_mlp.up_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.mlp.switch_mlp.up_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.gate_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.gate_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.gate_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.k_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.k_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.o_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.o_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.q_proj.biases": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.q_proj.scales": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.v_experts.biases": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.v_experts.scales": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.v_experts.weight": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.v_router.bias": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.v_router.biases": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.v_router.scales": "model-00016-of-00048.safetensors", + "model.layers.15.self_attn.v_router.weight": "model-00016-of-00048.safetensors", + "model.layers.16.input_layernorm.weight": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.gate.bias": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.gate.biases": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.gate.scales": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.gate.weight": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.shared_experts.down_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.shared_experts.down_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.shared_experts.down_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.shared_experts.gate_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.shared_experts.gate_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.shared_experts.gate_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.shared_experts.up_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.shared_experts.up_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.shared_experts.up_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.switch_mlp.down_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.switch_mlp.down_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.switch_mlp.down_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.switch_mlp.gate_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.switch_mlp.gate_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.switch_mlp.gate_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.switch_mlp.up_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.switch_mlp.up_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.mlp.switch_mlp.up_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.gate_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.gate_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.gate_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.k_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.k_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.o_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.o_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.q_proj.biases": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.q_proj.scales": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.v_experts.biases": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.v_experts.scales": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.v_experts.weight": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.v_router.bias": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.v_router.biases": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.v_router.scales": "model-00017-of-00048.safetensors", + "model.layers.16.self_attn.v_router.weight": "model-00017-of-00048.safetensors", + "model.layers.17.input_layernorm.weight": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.gate.bias": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.gate.biases": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.gate.scales": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.gate.weight": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.shared_experts.down_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.shared_experts.down_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.shared_experts.down_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.shared_experts.gate_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.shared_experts.gate_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.shared_experts.gate_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.shared_experts.up_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.shared_experts.up_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.shared_experts.up_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.switch_mlp.down_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.switch_mlp.down_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.switch_mlp.down_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.switch_mlp.gate_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.switch_mlp.gate_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.switch_mlp.gate_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.switch_mlp.up_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.switch_mlp.up_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.mlp.switch_mlp.up_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.gate_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.gate_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.gate_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.k_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.k_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.o_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.o_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.q_proj.biases": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.q_proj.scales": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.v_experts.biases": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.v_experts.scales": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.v_experts.weight": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.v_router.bias": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.v_router.biases": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.v_router.scales": "model-00018-of-00048.safetensors", + "model.layers.17.self_attn.v_router.weight": "model-00018-of-00048.safetensors", + "model.layers.18.input_layernorm.weight": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.gate.bias": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.gate.biases": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.gate.scales": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.gate.weight": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.shared_experts.down_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.shared_experts.down_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.shared_experts.down_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.shared_experts.gate_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.shared_experts.gate_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.shared_experts.gate_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.shared_experts.up_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.shared_experts.up_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.shared_experts.up_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.switch_mlp.down_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.switch_mlp.down_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.switch_mlp.down_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.switch_mlp.gate_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.switch_mlp.gate_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.switch_mlp.gate_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.switch_mlp.up_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.switch_mlp.up_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.mlp.switch_mlp.up_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.gate_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.gate_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.gate_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.k_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.k_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.o_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.o_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.q_proj.biases": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.q_proj.scales": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.v_experts.biases": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.v_experts.scales": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.v_experts.weight": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.v_router.bias": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.v_router.biases": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.v_router.scales": "model-00019-of-00048.safetensors", + "model.layers.18.self_attn.v_router.weight": "model-00019-of-00048.safetensors", + "model.layers.19.input_layernorm.weight": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.gate.bias": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.gate.biases": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.gate.scales": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.gate.weight": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.shared_experts.down_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.shared_experts.down_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.shared_experts.down_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.shared_experts.gate_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.shared_experts.gate_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.shared_experts.gate_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.shared_experts.up_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.shared_experts.up_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.shared_experts.up_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.switch_mlp.down_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.switch_mlp.down_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.switch_mlp.down_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.switch_mlp.gate_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.switch_mlp.gate_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.switch_mlp.gate_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.switch_mlp.up_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.switch_mlp.up_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.mlp.switch_mlp.up_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.gate_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.gate_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.gate_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.k_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.k_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.o_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.o_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.q_proj.biases": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.q_proj.scales": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.v_experts.biases": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.v_experts.scales": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.v_experts.weight": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.v_router.bias": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.v_router.biases": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.v_router.scales": "model-00020-of-00048.safetensors", + "model.layers.19.self_attn.v_router.weight": "model-00020-of-00048.safetensors", + "model.layers.2.input_layernorm.weight": "model-00003-of-00048.safetensors", + "model.layers.2.mlp.down_proj.biases": "model-00003-of-00048.safetensors", + "model.layers.2.mlp.down_proj.scales": "model-00003-of-00048.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00003-of-00048.safetensors", + "model.layers.2.mlp.gate_proj.biases": "model-00003-of-00048.safetensors", + "model.layers.2.mlp.gate_proj.scales": "model-00003-of-00048.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00003-of-00048.safetensors", + "model.layers.2.mlp.up_proj.biases": "model-00003-of-00048.safetensors", + "model.layers.2.mlp.up_proj.scales": "model-00003-of-00048.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00003-of-00048.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.gate_proj.biases": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.gate_proj.scales": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.gate_proj.weight": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.k_proj.biases": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.k_proj.scales": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.o_proj.biases": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.o_proj.scales": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.q_proj.biases": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.q_proj.scales": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.v_proj.biases": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.v_proj.scales": "model-00003-of-00048.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00003-of-00048.safetensors", + "model.layers.20.input_layernorm.weight": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.gate.bias": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.gate.biases": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.gate.scales": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.gate.weight": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.shared_experts.down_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.shared_experts.down_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.shared_experts.down_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.shared_experts.gate_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.shared_experts.gate_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.shared_experts.gate_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.shared_experts.up_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.shared_experts.up_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.shared_experts.up_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.switch_mlp.down_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.switch_mlp.down_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.switch_mlp.down_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.switch_mlp.gate_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.switch_mlp.gate_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.switch_mlp.gate_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.switch_mlp.up_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.switch_mlp.up_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.mlp.switch_mlp.up_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.gate_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.gate_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.gate_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.k_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.k_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.o_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.o_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.q_proj.biases": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.q_proj.scales": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.v_experts.biases": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.v_experts.scales": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.v_experts.weight": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.v_router.bias": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.v_router.biases": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.v_router.scales": "model-00021-of-00048.safetensors", + "model.layers.20.self_attn.v_router.weight": "model-00021-of-00048.safetensors", + "model.layers.21.input_layernorm.weight": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.gate.bias": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.gate.biases": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.gate.scales": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.gate.weight": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.shared_experts.down_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.shared_experts.down_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.shared_experts.down_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.shared_experts.gate_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.shared_experts.gate_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.shared_experts.gate_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.shared_experts.up_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.shared_experts.up_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.shared_experts.up_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.switch_mlp.down_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.switch_mlp.down_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.switch_mlp.down_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.switch_mlp.gate_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.switch_mlp.gate_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.switch_mlp.gate_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.switch_mlp.up_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.switch_mlp.up_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.mlp.switch_mlp.up_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.gate_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.gate_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.gate_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.k_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.k_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.o_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.o_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.q_proj.biases": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.q_proj.scales": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.v_experts.biases": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.v_experts.scales": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.v_experts.weight": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.v_router.bias": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.v_router.biases": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.v_router.scales": "model-00022-of-00048.safetensors", + "model.layers.21.self_attn.v_router.weight": "model-00022-of-00048.safetensors", + "model.layers.22.input_layernorm.weight": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.gate.bias": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.gate.biases": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.gate.scales": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.gate.weight": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.shared_experts.down_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.shared_experts.down_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.shared_experts.down_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.shared_experts.gate_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.shared_experts.gate_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.shared_experts.gate_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.shared_experts.up_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.shared_experts.up_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.shared_experts.up_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.switch_mlp.down_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.switch_mlp.down_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.switch_mlp.down_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.switch_mlp.gate_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.switch_mlp.gate_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.switch_mlp.gate_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.switch_mlp.up_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.switch_mlp.up_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.mlp.switch_mlp.up_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.gate_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.gate_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.gate_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.k_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.k_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.o_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.o_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.q_proj.biases": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.q_proj.scales": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.v_experts.biases": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.v_experts.scales": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.v_experts.weight": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.v_router.bias": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.v_router.biases": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.v_router.scales": "model-00023-of-00048.safetensors", + "model.layers.22.self_attn.v_router.weight": "model-00023-of-00048.safetensors", + "model.layers.23.input_layernorm.weight": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.gate.bias": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.gate.biases": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.gate.scales": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.gate.weight": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.shared_experts.down_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.shared_experts.down_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.shared_experts.down_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.shared_experts.gate_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.shared_experts.gate_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.shared_experts.gate_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.shared_experts.up_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.shared_experts.up_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.shared_experts.up_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.switch_mlp.down_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.switch_mlp.down_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.switch_mlp.down_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.switch_mlp.gate_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.switch_mlp.gate_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.switch_mlp.gate_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.switch_mlp.up_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.switch_mlp.up_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.mlp.switch_mlp.up_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.gate_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.gate_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.gate_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.k_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.k_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.o_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.o_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.q_proj.biases": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.q_proj.scales": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.v_experts.biases": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.v_experts.scales": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.v_experts.weight": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.v_router.bias": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.v_router.biases": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.v_router.scales": "model-00024-of-00048.safetensors", + "model.layers.23.self_attn.v_router.weight": "model-00024-of-00048.safetensors", + "model.layers.24.input_layernorm.weight": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.gate.bias": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.gate.biases": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.gate.scales": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.gate.weight": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.shared_experts.down_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.shared_experts.down_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.shared_experts.down_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.shared_experts.gate_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.shared_experts.gate_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.shared_experts.gate_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.shared_experts.up_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.shared_experts.up_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.shared_experts.up_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.switch_mlp.down_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.switch_mlp.down_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.switch_mlp.down_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.switch_mlp.gate_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.switch_mlp.gate_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.switch_mlp.gate_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.switch_mlp.up_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.switch_mlp.up_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.mlp.switch_mlp.up_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.gate_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.gate_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.gate_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.k_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.k_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.o_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.o_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.q_proj.biases": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.q_proj.scales": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.v_experts.biases": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.v_experts.scales": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.v_experts.weight": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.v_router.bias": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.v_router.biases": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.v_router.scales": "model-00025-of-00048.safetensors", + "model.layers.24.self_attn.v_router.weight": "model-00025-of-00048.safetensors", + "model.layers.25.input_layernorm.weight": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.gate.bias": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.gate.biases": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.gate.scales": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.gate.weight": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.shared_experts.down_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.shared_experts.down_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.shared_experts.down_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.shared_experts.gate_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.shared_experts.gate_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.shared_experts.gate_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.shared_experts.up_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.shared_experts.up_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.shared_experts.up_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.switch_mlp.down_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.switch_mlp.down_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.switch_mlp.down_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.switch_mlp.gate_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.switch_mlp.gate_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.switch_mlp.gate_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.switch_mlp.up_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.switch_mlp.up_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.mlp.switch_mlp.up_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.gate_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.gate_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.gate_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.k_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.k_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.o_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.o_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.q_proj.biases": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.q_proj.scales": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.v_experts.biases": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.v_experts.scales": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.v_experts.weight": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.v_router.bias": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.v_router.biases": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.v_router.scales": "model-00026-of-00048.safetensors", + "model.layers.25.self_attn.v_router.weight": "model-00026-of-00048.safetensors", + "model.layers.26.input_layernorm.weight": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.gate.bias": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.gate.biases": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.gate.scales": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.gate.weight": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.shared_experts.down_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.shared_experts.down_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.shared_experts.down_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.shared_experts.gate_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.shared_experts.gate_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.shared_experts.gate_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.shared_experts.up_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.shared_experts.up_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.shared_experts.up_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.switch_mlp.down_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.switch_mlp.down_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.switch_mlp.down_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.switch_mlp.gate_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.switch_mlp.gate_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.switch_mlp.gate_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.switch_mlp.up_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.switch_mlp.up_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.mlp.switch_mlp.up_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.gate_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.gate_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.gate_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.k_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.k_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.o_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.o_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.q_proj.biases": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.q_proj.scales": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.v_experts.biases": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.v_experts.scales": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.v_experts.weight": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.v_router.bias": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.v_router.biases": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.v_router.scales": "model-00027-of-00048.safetensors", + "model.layers.26.self_attn.v_router.weight": "model-00027-of-00048.safetensors", + "model.layers.27.input_layernorm.weight": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.gate.bias": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.gate.biases": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.gate.scales": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.gate.weight": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.shared_experts.down_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.shared_experts.down_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.shared_experts.down_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.shared_experts.gate_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.shared_experts.gate_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.shared_experts.gate_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.shared_experts.up_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.shared_experts.up_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.shared_experts.up_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.switch_mlp.down_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.switch_mlp.down_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.switch_mlp.down_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.switch_mlp.gate_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.switch_mlp.gate_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.switch_mlp.gate_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.switch_mlp.up_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.switch_mlp.up_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.mlp.switch_mlp.up_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.gate_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.gate_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.gate_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.k_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.k_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.o_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.o_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.q_proj.biases": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.q_proj.scales": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.v_experts.biases": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.v_experts.scales": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.v_experts.weight": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.v_router.bias": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.v_router.biases": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.v_router.scales": "model-00028-of-00048.safetensors", + "model.layers.27.self_attn.v_router.weight": "model-00028-of-00048.safetensors", + "model.layers.28.input_layernorm.weight": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.gate.bias": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.gate.biases": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.gate.scales": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.gate.weight": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.shared_experts.down_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.shared_experts.down_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.shared_experts.down_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.shared_experts.gate_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.shared_experts.gate_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.shared_experts.gate_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.shared_experts.up_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.shared_experts.up_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.shared_experts.up_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.switch_mlp.down_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.switch_mlp.down_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.switch_mlp.down_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.switch_mlp.gate_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.switch_mlp.gate_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.switch_mlp.gate_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.switch_mlp.up_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.switch_mlp.up_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.mlp.switch_mlp.up_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.gate_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.gate_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.gate_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.k_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.k_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.o_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.o_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.q_proj.biases": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.q_proj.scales": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.v_experts.biases": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.v_experts.scales": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.v_experts.weight": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.v_router.bias": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.v_router.biases": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.v_router.scales": "model-00029-of-00048.safetensors", + "model.layers.28.self_attn.v_router.weight": "model-00029-of-00048.safetensors", + "model.layers.29.input_layernorm.weight": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.gate.bias": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.gate.biases": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.gate.scales": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.gate.weight": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.shared_experts.down_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.shared_experts.down_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.shared_experts.down_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.shared_experts.gate_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.shared_experts.gate_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.shared_experts.gate_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.shared_experts.up_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.shared_experts.up_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.shared_experts.up_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.switch_mlp.down_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.switch_mlp.down_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.switch_mlp.down_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.switch_mlp.gate_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.switch_mlp.gate_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.switch_mlp.gate_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.switch_mlp.up_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.switch_mlp.up_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.mlp.switch_mlp.up_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.gate_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.gate_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.gate_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.k_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.k_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.o_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.o_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.q_proj.biases": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.q_proj.scales": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.v_experts.biases": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.v_experts.scales": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.v_experts.weight": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.v_router.bias": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.v_router.biases": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.v_router.scales": "model-00030-of-00048.safetensors", + "model.layers.29.self_attn.v_router.weight": "model-00030-of-00048.safetensors", + "model.layers.3.input_layernorm.weight": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.gate.bias": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.gate.biases": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.gate.scales": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.gate.weight": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.shared_experts.down_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.shared_experts.down_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.shared_experts.down_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.shared_experts.gate_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.shared_experts.gate_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.shared_experts.gate_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.shared_experts.up_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.shared_experts.up_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.shared_experts.up_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.switch_mlp.down_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.switch_mlp.down_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.switch_mlp.down_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.switch_mlp.gate_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.switch_mlp.gate_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.switch_mlp.gate_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.switch_mlp.up_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.switch_mlp.up_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.mlp.switch_mlp.up_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.gate_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.gate_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.gate_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.k_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.k_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.o_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.o_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.q_proj.biases": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.q_proj.scales": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.v_experts.biases": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.v_experts.scales": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.v_experts.weight": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.v_router.bias": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.v_router.biases": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.v_router.scales": "model-00004-of-00048.safetensors", + "model.layers.3.self_attn.v_router.weight": "model-00004-of-00048.safetensors", + "model.layers.30.input_layernorm.weight": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.gate.bias": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.gate.biases": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.gate.scales": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.gate.weight": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.shared_experts.down_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.shared_experts.down_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.shared_experts.down_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.shared_experts.gate_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.shared_experts.gate_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.shared_experts.gate_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.shared_experts.up_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.shared_experts.up_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.shared_experts.up_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.switch_mlp.down_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.switch_mlp.down_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.switch_mlp.down_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.switch_mlp.gate_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.switch_mlp.gate_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.switch_mlp.gate_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.switch_mlp.up_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.switch_mlp.up_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.mlp.switch_mlp.up_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.gate_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.gate_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.gate_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.k_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.k_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.o_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.o_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.q_proj.biases": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.q_proj.scales": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.v_experts.biases": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.v_experts.scales": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.v_experts.weight": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.v_router.bias": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.v_router.biases": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.v_router.scales": "model-00031-of-00048.safetensors", + "model.layers.30.self_attn.v_router.weight": "model-00031-of-00048.safetensors", + "model.layers.31.input_layernorm.weight": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.gate.bias": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.gate.biases": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.gate.scales": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.gate.weight": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.shared_experts.down_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.shared_experts.down_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.shared_experts.down_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.shared_experts.gate_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.shared_experts.gate_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.shared_experts.gate_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.shared_experts.up_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.shared_experts.up_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.shared_experts.up_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.switch_mlp.down_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.switch_mlp.down_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.switch_mlp.down_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.switch_mlp.gate_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.switch_mlp.gate_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.switch_mlp.gate_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.switch_mlp.up_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.switch_mlp.up_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.mlp.switch_mlp.up_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.gate_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.gate_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.gate_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.k_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.k_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.o_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.o_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.q_proj.biases": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.q_proj.scales": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.v_experts.biases": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.v_experts.scales": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.v_experts.weight": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.v_router.bias": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.v_router.biases": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.v_router.scales": "model-00032-of-00048.safetensors", + "model.layers.31.self_attn.v_router.weight": "model-00032-of-00048.safetensors", + "model.layers.32.input_layernorm.weight": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.gate.bias": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.gate.biases": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.gate.scales": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.gate.weight": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.shared_experts.down_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.shared_experts.down_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.shared_experts.down_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.shared_experts.gate_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.shared_experts.gate_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.shared_experts.gate_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.shared_experts.up_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.shared_experts.up_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.shared_experts.up_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.switch_mlp.down_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.switch_mlp.down_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.switch_mlp.down_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.switch_mlp.gate_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.switch_mlp.gate_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.switch_mlp.gate_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.switch_mlp.up_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.switch_mlp.up_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.mlp.switch_mlp.up_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.gate_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.gate_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.gate_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.k_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.k_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.o_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.o_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.q_proj.biases": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.q_proj.scales": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.v_experts.biases": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.v_experts.scales": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.v_experts.weight": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.v_router.bias": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.v_router.biases": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.v_router.scales": "model-00033-of-00048.safetensors", + "model.layers.32.self_attn.v_router.weight": "model-00033-of-00048.safetensors", + "model.layers.33.input_layernorm.weight": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.gate.bias": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.gate.biases": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.gate.scales": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.gate.weight": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.shared_experts.down_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.shared_experts.down_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.shared_experts.down_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.shared_experts.gate_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.shared_experts.gate_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.shared_experts.gate_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.shared_experts.up_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.shared_experts.up_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.shared_experts.up_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.switch_mlp.down_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.switch_mlp.down_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.switch_mlp.down_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.switch_mlp.gate_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.switch_mlp.gate_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.switch_mlp.gate_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.switch_mlp.up_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.switch_mlp.up_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.mlp.switch_mlp.up_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.gate_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.gate_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.gate_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.k_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.k_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.o_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.o_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.q_proj.biases": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.q_proj.scales": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.v_experts.biases": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.v_experts.scales": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.v_experts.weight": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.v_router.bias": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.v_router.biases": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.v_router.scales": "model-00034-of-00048.safetensors", + "model.layers.33.self_attn.v_router.weight": "model-00034-of-00048.safetensors", + "model.layers.34.input_layernorm.weight": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.gate.bias": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.gate.biases": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.gate.scales": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.gate.weight": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.shared_experts.down_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.shared_experts.down_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.shared_experts.down_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.shared_experts.gate_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.shared_experts.gate_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.shared_experts.gate_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.shared_experts.up_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.shared_experts.up_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.shared_experts.up_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.switch_mlp.down_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.switch_mlp.down_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.switch_mlp.down_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.switch_mlp.gate_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.switch_mlp.gate_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.switch_mlp.gate_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.switch_mlp.up_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.switch_mlp.up_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.mlp.switch_mlp.up_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.gate_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.gate_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.gate_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.k_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.k_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.o_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.o_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.q_proj.biases": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.q_proj.scales": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.v_experts.biases": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.v_experts.scales": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.v_experts.weight": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.v_router.bias": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.v_router.biases": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.v_router.scales": "model-00035-of-00048.safetensors", + "model.layers.34.self_attn.v_router.weight": "model-00035-of-00048.safetensors", + "model.layers.35.input_layernorm.weight": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.gate.bias": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.gate.biases": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.gate.scales": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.gate.weight": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.shared_experts.down_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.shared_experts.down_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.shared_experts.down_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.shared_experts.gate_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.shared_experts.gate_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.shared_experts.gate_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.shared_experts.up_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.shared_experts.up_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.shared_experts.up_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.switch_mlp.down_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.switch_mlp.down_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.switch_mlp.down_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.switch_mlp.gate_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.switch_mlp.gate_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.switch_mlp.gate_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.switch_mlp.up_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.switch_mlp.up_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.mlp.switch_mlp.up_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.gate_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.gate_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.gate_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.k_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.k_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.o_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.o_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.q_proj.biases": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.q_proj.scales": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.v_experts.biases": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.v_experts.scales": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.v_experts.weight": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.v_router.bias": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.v_router.biases": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.v_router.scales": "model-00036-of-00048.safetensors", + "model.layers.35.self_attn.v_router.weight": "model-00036-of-00048.safetensors", + "model.layers.36.input_layernorm.weight": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.gate.bias": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.gate.biases": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.gate.scales": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.gate.weight": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.shared_experts.down_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.shared_experts.down_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.shared_experts.down_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.shared_experts.gate_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.shared_experts.gate_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.shared_experts.gate_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.shared_experts.up_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.shared_experts.up_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.shared_experts.up_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.switch_mlp.down_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.switch_mlp.down_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.switch_mlp.down_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.switch_mlp.gate_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.switch_mlp.gate_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.switch_mlp.gate_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.switch_mlp.up_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.switch_mlp.up_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.mlp.switch_mlp.up_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.post_attention_layernorm.weight": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.gate_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.gate_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.gate_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.k_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.k_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.k_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.o_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.o_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.o_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.q_proj.biases": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.q_proj.scales": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.q_proj.weight": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.v_experts.biases": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.v_experts.scales": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.v_experts.weight": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.v_router.bias": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.v_router.biases": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.v_router.scales": "model-00037-of-00048.safetensors", + "model.layers.36.self_attn.v_router.weight": "model-00037-of-00048.safetensors", + "model.layers.37.input_layernorm.weight": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.gate.bias": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.gate.biases": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.gate.scales": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.gate.weight": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.shared_experts.down_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.shared_experts.down_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.shared_experts.down_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.shared_experts.gate_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.shared_experts.gate_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.shared_experts.gate_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.shared_experts.up_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.shared_experts.up_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.shared_experts.up_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.switch_mlp.down_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.switch_mlp.down_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.switch_mlp.down_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.switch_mlp.gate_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.switch_mlp.gate_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.switch_mlp.gate_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.switch_mlp.up_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.switch_mlp.up_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.mlp.switch_mlp.up_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.post_attention_layernorm.weight": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.gate_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.gate_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.gate_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.k_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.k_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.k_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.o_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.o_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.o_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.q_proj.biases": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.q_proj.scales": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.q_proj.weight": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.v_experts.biases": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.v_experts.scales": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.v_experts.weight": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.v_router.bias": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.v_router.biases": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.v_router.scales": "model-00038-of-00048.safetensors", + "model.layers.37.self_attn.v_router.weight": "model-00038-of-00048.safetensors", + "model.layers.38.input_layernorm.weight": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.gate.bias": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.gate.biases": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.gate.scales": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.gate.weight": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.shared_experts.down_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.shared_experts.down_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.shared_experts.down_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.shared_experts.gate_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.shared_experts.gate_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.shared_experts.gate_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.shared_experts.up_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.shared_experts.up_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.shared_experts.up_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.switch_mlp.down_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.switch_mlp.down_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.switch_mlp.down_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.switch_mlp.gate_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.switch_mlp.gate_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.switch_mlp.gate_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.switch_mlp.up_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.switch_mlp.up_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.mlp.switch_mlp.up_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.post_attention_layernorm.weight": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.gate_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.gate_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.gate_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.k_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.k_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.k_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.o_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.o_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.o_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.q_proj.biases": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.q_proj.scales": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.q_proj.weight": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.v_experts.biases": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.v_experts.scales": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.v_experts.weight": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.v_router.bias": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.v_router.biases": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.v_router.scales": "model-00039-of-00048.safetensors", + "model.layers.38.self_attn.v_router.weight": "model-00039-of-00048.safetensors", + "model.layers.39.input_layernorm.weight": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.gate.bias": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.gate.biases": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.gate.scales": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.gate.weight": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.shared_experts.down_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.shared_experts.down_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.shared_experts.down_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.shared_experts.gate_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.shared_experts.gate_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.shared_experts.gate_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.shared_experts.up_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.shared_experts.up_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.shared_experts.up_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.switch_mlp.down_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.switch_mlp.down_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.switch_mlp.down_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.switch_mlp.gate_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.switch_mlp.gate_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.switch_mlp.gate_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.switch_mlp.up_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.switch_mlp.up_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.mlp.switch_mlp.up_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.post_attention_layernorm.weight": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.gate_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.gate_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.gate_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.k_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.k_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.k_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.o_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.o_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.o_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.q_proj.biases": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.q_proj.scales": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.q_proj.weight": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.v_experts.biases": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.v_experts.scales": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.v_experts.weight": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.v_router.bias": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.v_router.biases": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.v_router.scales": "model-00040-of-00048.safetensors", + "model.layers.39.self_attn.v_router.weight": "model-00040-of-00048.safetensors", + "model.layers.4.input_layernorm.weight": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.gate.bias": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.gate.biases": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.gate.scales": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.gate.weight": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.shared_experts.down_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.shared_experts.down_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.shared_experts.down_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.shared_experts.gate_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.shared_experts.gate_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.shared_experts.gate_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.shared_experts.up_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.shared_experts.up_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.shared_experts.up_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.switch_mlp.down_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.switch_mlp.down_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.switch_mlp.down_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.switch_mlp.gate_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.switch_mlp.gate_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.switch_mlp.gate_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.switch_mlp.up_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.switch_mlp.up_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.mlp.switch_mlp.up_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.gate_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.gate_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.gate_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.k_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.k_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.o_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.o_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.q_proj.biases": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.q_proj.scales": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.v_experts.biases": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.v_experts.scales": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.v_experts.weight": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.v_router.bias": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.v_router.biases": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.v_router.scales": "model-00005-of-00048.safetensors", + "model.layers.4.self_attn.v_router.weight": "model-00005-of-00048.safetensors", + "model.layers.40.input_layernorm.weight": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.gate.bias": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.gate.biases": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.gate.scales": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.gate.weight": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.shared_experts.down_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.shared_experts.down_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.shared_experts.down_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.shared_experts.gate_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.shared_experts.gate_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.shared_experts.gate_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.shared_experts.up_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.shared_experts.up_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.shared_experts.up_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.switch_mlp.down_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.switch_mlp.down_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.switch_mlp.down_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.switch_mlp.gate_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.switch_mlp.gate_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.switch_mlp.gate_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.switch_mlp.up_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.switch_mlp.up_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.mlp.switch_mlp.up_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.post_attention_layernorm.weight": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.gate_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.gate_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.gate_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.k_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.k_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.k_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.o_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.o_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.o_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.q_proj.biases": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.q_proj.scales": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.q_proj.weight": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.v_experts.biases": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.v_experts.scales": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.v_experts.weight": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.v_router.bias": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.v_router.biases": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.v_router.scales": "model-00041-of-00048.safetensors", + "model.layers.40.self_attn.v_router.weight": "model-00041-of-00048.safetensors", + "model.layers.41.input_layernorm.weight": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.gate.bias": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.gate.biases": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.gate.scales": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.gate.weight": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.shared_experts.down_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.shared_experts.down_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.shared_experts.down_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.shared_experts.gate_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.shared_experts.gate_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.shared_experts.gate_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.shared_experts.up_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.shared_experts.up_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.shared_experts.up_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.switch_mlp.down_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.switch_mlp.down_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.switch_mlp.down_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.switch_mlp.gate_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.switch_mlp.gate_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.switch_mlp.gate_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.switch_mlp.up_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.switch_mlp.up_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.mlp.switch_mlp.up_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.post_attention_layernorm.weight": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.gate_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.gate_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.gate_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.k_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.k_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.k_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.o_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.o_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.o_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.q_proj.biases": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.q_proj.scales": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.q_proj.weight": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.v_experts.biases": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.v_experts.scales": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.v_experts.weight": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.v_router.bias": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.v_router.biases": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.v_router.scales": "model-00042-of-00048.safetensors", + "model.layers.41.self_attn.v_router.weight": "model-00042-of-00048.safetensors", + "model.layers.42.input_layernorm.weight": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.gate.bias": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.gate.biases": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.gate.scales": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.gate.weight": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.shared_experts.down_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.shared_experts.down_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.shared_experts.down_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.shared_experts.gate_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.shared_experts.gate_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.shared_experts.gate_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.shared_experts.up_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.shared_experts.up_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.shared_experts.up_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.switch_mlp.down_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.switch_mlp.down_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.switch_mlp.down_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.switch_mlp.gate_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.switch_mlp.gate_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.switch_mlp.gate_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.switch_mlp.up_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.switch_mlp.up_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.mlp.switch_mlp.up_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.post_attention_layernorm.weight": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.gate_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.gate_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.gate_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.k_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.k_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.k_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.o_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.o_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.o_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.q_proj.biases": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.q_proj.scales": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.q_proj.weight": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.v_experts.biases": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.v_experts.scales": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.v_experts.weight": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.v_router.bias": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.v_router.biases": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.v_router.scales": "model-00043-of-00048.safetensors", + "model.layers.42.self_attn.v_router.weight": "model-00043-of-00048.safetensors", + "model.layers.43.input_layernorm.weight": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.gate.bias": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.gate.biases": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.gate.scales": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.gate.weight": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.shared_experts.down_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.shared_experts.down_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.shared_experts.down_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.shared_experts.gate_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.shared_experts.gate_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.shared_experts.gate_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.shared_experts.up_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.shared_experts.up_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.shared_experts.up_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.switch_mlp.down_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.switch_mlp.down_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.switch_mlp.down_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.switch_mlp.gate_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.switch_mlp.gate_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.switch_mlp.gate_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.switch_mlp.up_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.switch_mlp.up_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.mlp.switch_mlp.up_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.post_attention_layernorm.weight": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.gate_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.gate_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.gate_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.k_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.k_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.k_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.o_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.o_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.o_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.q_proj.biases": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.q_proj.scales": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.q_proj.weight": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.v_experts.biases": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.v_experts.scales": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.v_experts.weight": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.v_router.bias": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.v_router.biases": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.v_router.scales": "model-00044-of-00048.safetensors", + "model.layers.43.self_attn.v_router.weight": "model-00044-of-00048.safetensors", + "model.layers.44.input_layernorm.weight": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.gate.bias": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.gate.biases": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.gate.scales": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.gate.weight": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.shared_experts.down_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.shared_experts.down_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.shared_experts.down_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.shared_experts.gate_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.shared_experts.gate_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.shared_experts.gate_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.shared_experts.up_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.shared_experts.up_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.shared_experts.up_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.switch_mlp.down_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.switch_mlp.down_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.switch_mlp.down_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.switch_mlp.gate_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.switch_mlp.gate_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.switch_mlp.gate_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.switch_mlp.up_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.switch_mlp.up_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.mlp.switch_mlp.up_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.post_attention_layernorm.weight": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.gate_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.gate_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.gate_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.k_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.k_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.k_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.o_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.o_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.o_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.q_proj.biases": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.q_proj.scales": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.q_proj.weight": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.v_experts.biases": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.v_experts.scales": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.v_experts.weight": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.v_router.bias": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.v_router.biases": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.v_router.scales": "model-00045-of-00048.safetensors", + "model.layers.44.self_attn.v_router.weight": "model-00045-of-00048.safetensors", + "model.layers.45.input_layernorm.weight": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.gate.bias": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.gate.biases": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.gate.scales": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.gate.weight": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.shared_experts.down_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.shared_experts.down_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.shared_experts.down_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.shared_experts.gate_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.shared_experts.gate_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.shared_experts.gate_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.shared_experts.up_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.shared_experts.up_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.shared_experts.up_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.switch_mlp.down_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.switch_mlp.down_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.switch_mlp.down_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.switch_mlp.gate_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.switch_mlp.gate_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.switch_mlp.gate_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.switch_mlp.up_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.switch_mlp.up_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.mlp.switch_mlp.up_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.post_attention_layernorm.weight": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.gate_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.gate_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.gate_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.k_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.k_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.k_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.o_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.o_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.o_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.q_proj.biases": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.q_proj.scales": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.q_proj.weight": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.v_experts.biases": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.v_experts.scales": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.v_experts.weight": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.v_router.bias": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.v_router.biases": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.v_router.scales": "model-00046-of-00048.safetensors", + "model.layers.45.self_attn.v_router.weight": "model-00046-of-00048.safetensors", + "model.layers.46.input_layernorm.weight": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.gate.bias": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.gate.biases": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.gate.scales": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.gate.weight": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.shared_experts.down_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.shared_experts.down_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.shared_experts.down_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.shared_experts.gate_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.shared_experts.gate_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.shared_experts.gate_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.shared_experts.up_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.shared_experts.up_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.shared_experts.up_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.switch_mlp.down_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.switch_mlp.down_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.switch_mlp.down_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.switch_mlp.gate_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.switch_mlp.gate_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.switch_mlp.gate_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.switch_mlp.up_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.switch_mlp.up_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.mlp.switch_mlp.up_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.post_attention_layernorm.weight": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.gate_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.gate_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.gate_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.k_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.k_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.k_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.o_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.o_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.o_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.q_proj.biases": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.q_proj.scales": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.q_proj.weight": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.v_experts.biases": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.v_experts.scales": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.v_experts.weight": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.v_router.bias": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.v_router.biases": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.v_router.scales": "model-00047-of-00048.safetensors", + "model.layers.46.self_attn.v_router.weight": "model-00047-of-00048.safetensors", + "model.layers.47.input_layernorm.weight": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.gate.bias": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.gate.biases": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.gate.scales": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.gate.weight": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.shared_experts.down_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.shared_experts.down_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.shared_experts.down_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.shared_experts.gate_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.shared_experts.gate_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.shared_experts.gate_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.shared_experts.up_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.shared_experts.up_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.shared_experts.up_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.switch_mlp.down_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.switch_mlp.down_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.switch_mlp.down_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.switch_mlp.gate_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.switch_mlp.gate_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.switch_mlp.gate_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.switch_mlp.up_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.switch_mlp.up_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.mlp.switch_mlp.up_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.post_attention_layernorm.weight": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.gate_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.gate_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.gate_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.k_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.k_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.k_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.o_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.o_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.o_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.q_proj.biases": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.q_proj.scales": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.q_proj.weight": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.v_experts.biases": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.v_experts.scales": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.v_experts.weight": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.v_router.bias": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.v_router.biases": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.v_router.scales": "model-00048-of-00048.safetensors", + "model.layers.47.self_attn.v_router.weight": "model-00048-of-00048.safetensors", + "model.layers.5.input_layernorm.weight": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.gate.bias": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.gate.biases": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.gate.scales": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.gate.weight": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.shared_experts.down_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.shared_experts.down_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.shared_experts.down_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.shared_experts.gate_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.shared_experts.gate_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.shared_experts.gate_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.shared_experts.up_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.shared_experts.up_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.shared_experts.up_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.switch_mlp.down_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.switch_mlp.down_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.switch_mlp.down_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.switch_mlp.gate_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.switch_mlp.gate_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.switch_mlp.gate_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.switch_mlp.up_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.switch_mlp.up_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.mlp.switch_mlp.up_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.gate_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.gate_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.gate_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.k_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.k_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.o_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.o_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.q_proj.biases": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.q_proj.scales": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.v_experts.biases": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.v_experts.scales": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.v_experts.weight": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.v_router.bias": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.v_router.biases": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.v_router.scales": "model-00006-of-00048.safetensors", + "model.layers.5.self_attn.v_router.weight": "model-00006-of-00048.safetensors", + "model.layers.6.input_layernorm.weight": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.gate.bias": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.gate.biases": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.gate.scales": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.gate.weight": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.shared_experts.down_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.shared_experts.down_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.shared_experts.down_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.shared_experts.gate_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.shared_experts.gate_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.shared_experts.gate_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.shared_experts.up_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.shared_experts.up_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.shared_experts.up_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.switch_mlp.down_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.switch_mlp.down_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.switch_mlp.down_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.switch_mlp.gate_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.switch_mlp.gate_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.switch_mlp.gate_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.switch_mlp.up_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.switch_mlp.up_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.mlp.switch_mlp.up_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.gate_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.gate_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.gate_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.k_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.k_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.o_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.o_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.q_proj.biases": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.q_proj.scales": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.v_experts.biases": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.v_experts.scales": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.v_experts.weight": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.v_router.bias": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.v_router.biases": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.v_router.scales": "model-00007-of-00048.safetensors", + "model.layers.6.self_attn.v_router.weight": "model-00007-of-00048.safetensors", + "model.layers.7.input_layernorm.weight": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.gate.bias": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.gate.biases": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.gate.scales": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.gate.weight": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.shared_experts.down_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.shared_experts.down_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.shared_experts.down_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.shared_experts.gate_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.shared_experts.gate_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.shared_experts.gate_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.shared_experts.up_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.shared_experts.up_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.shared_experts.up_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.switch_mlp.down_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.switch_mlp.down_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.switch_mlp.down_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.switch_mlp.gate_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.switch_mlp.gate_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.switch_mlp.gate_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.switch_mlp.up_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.switch_mlp.up_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.mlp.switch_mlp.up_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.gate_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.gate_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.gate_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.k_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.k_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.o_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.o_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.q_proj.biases": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.q_proj.scales": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.v_experts.biases": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.v_experts.scales": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.v_experts.weight": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.v_router.bias": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.v_router.biases": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.v_router.scales": "model-00008-of-00048.safetensors", + "model.layers.7.self_attn.v_router.weight": "model-00008-of-00048.safetensors", + "model.layers.8.input_layernorm.weight": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.gate.bias": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.gate.biases": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.gate.scales": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.gate.weight": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.shared_experts.down_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.shared_experts.down_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.shared_experts.down_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.shared_experts.gate_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.shared_experts.gate_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.shared_experts.gate_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.shared_experts.up_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.shared_experts.up_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.shared_experts.up_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.switch_mlp.down_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.switch_mlp.down_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.switch_mlp.down_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.switch_mlp.gate_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.switch_mlp.gate_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.switch_mlp.gate_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.switch_mlp.up_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.switch_mlp.up_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.mlp.switch_mlp.up_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.gate_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.gate_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.gate_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.k_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.k_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.o_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.o_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.q_proj.biases": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.q_proj.scales": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.v_experts.biases": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.v_experts.scales": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.v_experts.weight": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.v_router.bias": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.v_router.biases": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.v_router.scales": "model-00009-of-00048.safetensors", + "model.layers.8.self_attn.v_router.weight": "model-00009-of-00048.safetensors", + "model.layers.9.input_layernorm.weight": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.gate.bias": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.gate.biases": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.gate.scales": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.gate.weight": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.shared_experts.down_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.shared_experts.down_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.shared_experts.down_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.shared_experts.gate_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.shared_experts.gate_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.shared_experts.gate_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.shared_experts.up_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.shared_experts.up_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.shared_experts.up_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.switch_mlp.down_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.switch_mlp.down_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.switch_mlp.down_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.switch_mlp.gate_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.switch_mlp.gate_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.switch_mlp.gate_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.switch_mlp.up_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.switch_mlp.up_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.mlp.switch_mlp.up_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.gate_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.gate_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.gate_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.k_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.k_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.o_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.o_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.q_proj.biases": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.q_proj.scales": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.v_experts.biases": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.v_experts.scales": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.v_experts.weight": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.v_router.bias": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.v_router.biases": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.v_router.scales": "model-00010-of-00048.safetensors", + "model.layers.9.self_attn.v_router.weight": "model-00010-of-00048.safetensors", + "model.norm.weight": "model-00048-of-00048.safetensors" + } +} diff --git a/special_tokens_map.json b/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..2aead6f5631bda6d20502ec6a6fbe91fa0b07106 --- /dev/null +++ b/special_tokens_map.json @@ -0,0 +1,4 @@ +{ + "bos_token": "<|ifm|begin_of_text|>", + "eos_token": "<|ifm|endoftext|>" +} diff --git a/tokenizer.json b/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..8b2e336766221d496bbc21b58dcb83b9b9b9be4e --- /dev/null +++ b/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fa69519ff1ee3ee55ac50927b00058e88b9eb90802fabaf3c6b05804601c53b +size 29197477 diff --git a/tokenizer_config.json b/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..89e287197c36817940da96ce136ed1217212f420 --- /dev/null +++ b/tokenizer_config.json @@ -0,0 +1,8 @@ +{ + "bos_token": "<|ifm|begin_of_text|>", + "clean_up_tokenization_spaces": false, + "eos_token": "<|ifm|endoftext|>", + "extra_special_tokens": {}, + "model_max_length": 1000000000000000019884624838656, + "tokenizer_class": "PreTrainedTokenizerFast" +}