| active_expert_width: null |
| active_num_experts: null |
| attention_dropout: 0.0 |
| attention_type: gqa |
| causal_context_fusion_size: 0 |
| causal_context_gate_init: 1.0 |
| causal_contextual_mix_init: 0.0 |
| causal_conv_dilation_cycle: 8 |
| causal_conv_kernel_size: 4 |
| causal_delta_chunk_size: 512 |
| causal_delta_collision_normalized: false |
| causal_delta_lexical_forge: false |
| causal_delta_lexical_values: false |
| causal_delta_occurrence_address: false |
| causal_delta_timescales: 1 |
| causal_stable_delta: false |
| causal_state_rank: 16 |
| collect_moe_telemetry: false |
| expert_initialization: gpt_normal |
| extra_config: {} |
| hash_channel_scale_init: 0.0 |
| hash_pair_gate_init: 0.5 |
| hidden_act: silu |
| hidden_size: 896 |
| initializer_range: 0.02 |
| intermediate_size: 256 |
| is_causal: true |
| learn_hash_channel_modulation: false |
| learn_hash_pair_gates: false |
| lexical_gqa_gate_init: 0.0 |
| lexical_gqa_rank: 16 |
| lexical_gqa_use_token_code: true |
| lexical_key_gate_init: 0.05 |
| lexical_object_gate_init: 0.1 |
| lexical_object_rank: 16 |
| lexical_zipf_alpha: 0.25 |
| lexical_zipf_floor: 0.1 |
| lexical_zipf_mode: uniform |
| lexical_zipf_path: null |
| lexical_zipf_permutation_seed: 1729 |
| lsh_bits: 0 |
| lsh_from_layer: 0 |
| lsh_routing: false |
| lsh_threshold_mode: zero |
| max_position_embeddings: 2048 |
| micro_expert_gate_init: 0.1 |
| micro_expert_width: 16 |
| micro_num_experts: 4 |
| mlp_type: tr_hash_engine |
| mup_base_width: 256 |
| norm_eps: 1.0e-06 |
| norm_type: rmsnorm |
| num_attention_heads: 14 |
| num_experts: 4 |
| num_hidden_layers: 16 |
| num_key_value_heads: 2 |
| rope_fraction: 1.0 |
| rope_theta: 10000.0 |
| rope_type: standard |
| route_hash_count: 2 |
| routed_gate_init: 1.0 |
| routed_output_scale: 2.0 |
| routed_output_scale_first_layer: null |
| routed_output_scale_last_layer: null |
| routing_strategy: token_id_multi_hash |
| shared_expert: true |
| shared_expert_chunk_tokens: 0 |
| shared_gate_init: 1.0 |
| shared_intermediate_size: 3072 |
| shared_output_scale: 1.0 |
| shared_output_scale_first_layer: null |
| shared_output_scale_last_layer: null |
| sliding_window: null |
| static_expert_capacity: false |
| tie_lexical_object_embeddings: false |
| tie_word_embeddings: true |
| token_frequencies: null |
| top_k: 2 |
| top_k_primary_weight: 0.5 |
| tr_mha_adapter_gate_init: 0.1 |
| tr_mha_adapter_rank: 8 |
| tr_mha_id_other_logit: -2.0 |
| tr_mha_id_primary_logit: 2.0 |
| tr_mha_id_secondary_logit: 1.0 |
| tr_mha_num_experts: 4 |
| tr_mha_targets: qv |
| tr_mha_top_k: 2 |
| tr_mha_verifier_gate_init: 0.1 |
| tr_mha_verifier_temperature: 1.0 |
| use_cache: true |
| use_cggr: auto |
| use_custom_kernels: true |
| use_mup_attn_scale: false |
| use_mup_init: false |
| use_mup_output_mult: false |
| use_qk_norm: true |
| use_sdpa: true |
| use_shared_routed_gates: false |
| vocab_size: 32000 |
|
|
