{
"problems": [
{
"board": "L1",
"id": "001_attention_softmax_dropout_value_matmul_backward",
"name": "Attention softmax dropout value matmul backward",
"group": "Backward pass",
"op": "attention softmax + dropout + PV backward (GQA 80/8/128)",
"direction": "backward",
"model": "ByteDance/Seed-Seed-OSS-36B-Instruct",
"workloads": 16
},
{
"board": "L1",
"id": "002_vae_conv3x3_groupnorm_silu_residual_fused",
"name": "VAE conv3x3 GroupNorm SiLU residual fused",
"group": "Image and video",
"op": "VAE conv3x3 + GroupNorm + SiLU residual block",
"direction": "forward",
"model": "Efficient-Large-Model/Sana-Sprint-1.6B-1024px",
"workloads": 20
},
{
"board": "L1",
"id": "003_lm_head_projection_with_logit_slicing",
"name": "LM head projection with logit slicing",
"group": "Projections and GEMM",
"op": "LM head GEMM on the last N positions (logit slicing)",
"direction": "forward",
"model": "LGAI-EXAONE/EXAONE-4.0-1.2B",
"workloads": 16
},
{
"board": "L1",
"id": "004_attention_output_projection_with_reshape_backward",
"name": "Attention output projection with reshape backward",
"group": "Backward pass",
"op": "attention o_proj + reshape backward",
"direction": "backward",
"model": "LiquidAI/LFM2-1.2B",
"workloads": 16
},
{
"board": "L1",
"id": "005_conv_gated_projection_with_causal_conv",
"name": "Conv gated projection with causal conv",
"group": "State space and conv",
"op": "LFM2 short-conv block: in-proj, gate, causal depthwise conv1d, out-proj",
"direction": "forward",
"model": "LiquidAI/LFM2-1.2B",
"workloads": 16
},
{
"board": "L1",
"id": "006_hyena_depthwise_conv1d_split_gate",
"name": "Hyena depthwise Conv1D split gate",
"group": "State space and conv",
"op": "Hyena depthwise conv1d + split + gate",
"direction": "forward",
"model": "LongSafari/hyenadna-small-32k-seqlen-hf",
"workloads": 16
},
{
"board": "L1",
"id": "007_hyena_fft_size_padding_rfft",
"name": "Hyena FFT size padding RFFT",
"group": "State space and conv",
"op": "Hyena FFT padding + rfft",
"direction": "forward",
"model": "LongSafari/hyenadna-small-32k-seqlen-hf",
"workloads": 16
},
{
"board": "L1",
"id": "008_expert_output_weighted_index_add_accumulation",
"name": "Expert output weighted index add accumulation",
"group": "MLP and MoE",
"op": "MoE combine: weighted index_add of expert outputs",
"direction": "forward",
"model": "MiniMaxAI/MiniMax-M2",
"workloads": 16
},
{
"board": "L1",
"id": "009_expert_token_scatter_with_weighted_forward_backward",
"name": "Expert token scatter with weighted forward backward",
"group": "Backward pass",
"op": "MoE token scatter, weighted, forward + backward",
"direction": "backward",
"model": "MiniMaxAI/MiniMax-M2",
"workloads": 16
},
{
"board": "L1",
"id": "010_attention_value_projection_with_transpose",
"name": "Attention value projection with transpose",
"group": "Projections and GEMM",
"op": "V projection (5120 -> 1024) + reshape / transpose",
"direction": "forward",
"model": "NousResearch/DeepHermes-3-Mistral-24B-Preview",
"workloads": 16
},
{
"board": "L1",
"id": "011_rotary_position_embedding",
"name": "Rotary position embedding",
"group": "Attention",
"op": "RoPE cos/sin from positions with llama3 scaling",
"direction": "forward",
"model": "NousResearch/Hermes-4-70B",
"workloads": 16
},
{
"board": "L1",
"id": "012_fused_cos_sin_embedding_generation",
"name": "Fused cos sin embedding generation",
"group": "Attention",
"op": "RoPE cos/sin from frequencies with attention scaling",
"direction": "forward",
"model": "NousResearch/Hermes-4-405B",
"workloads": 16
},
{
"board": "L1",
"id": "013_fused_residual_rms_norm_backward",
"name": "Fused residual RMS norm backward",
"group": "Backward pass",
"op": "fused residual + RMSNorm backward",
"direction": "backward",
"model": "Qwen/Qwen3-4B-Instruct-2507",
"workloads": 16
},
{
"board": "L1",
"id": "014_rotary_embedding_with_attention_scaling_backward",
"name": "Rotary embedding with attention scaling backward",
"group": "Backward pass",
"op": "RoPE with attention scaling backward",
"direction": "backward",
"model": "Qwen/Qwen3-4B-Instruct-2507",
"workloads": 16
},
{
"board": "L1",
"id": "015_grouped_query_attention_with_rope_and_qk_norm",
"name": "Grouped query attention with RoPE and QK norm",
"group": "Attention",
"op": "GQA attention block: QKV proj, QK-RMSNorm, RoPE, causal SDPA, o_proj",
"direction": "forward",
"model": "Qwen/Qwen3-8B",
"workloads": 16
},
{
"board": "L1",
"id": "016_rope_inverse_frequency_computation",
"name": "RoPE inverse frequency computation",
"group": "Attention",
"op": "RoPE inverse-frequency table",
"direction": "forward",
"model": "Qwen/Qwen3-8B",
"workloads": 16
},
{
"board": "L1",
"id": "017_moe_expert_swiglu_with_down_projection_backward",
"name": "MoE expert SwiGLU with down projection backward",
"group": "Backward pass",
"op": "MoE expert SwiGLU + down projection backward",
"direction": "backward",
"model": "Qwen/Qwen3-30B-A3B-Instruct-2507",
"workloads": 16
},
{
"board": "L1",
"id": "018_fused_rope_with_qk_norm_and_kv_cache_update",
"name": "Fused RoPE with QK norm and KV cache update",
"group": "Attention",
"op": "fused QK-RMSNorm + RoPE + KV-cache write",
"direction": "forward",
"model": "Qwen/Qwen3-Coder-480B-A35B-Instruct",
"workloads": 13
},
{
"board": "L1",
"id": "019_vision_3d_rotary_embedding_with_spatial_merge_indexing_backward",
"name": "Vision 3D rotary embedding with spatial merge indexing backward",
"group": "Backward pass",
"op": "vision 3D RoPE with spatial-merge indexing backward",
"direction": "backward",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"workloads": 16
},
{
"board": "L1",
"id": "020_vision_patch_merger_spatial_shuffle_mlp",
"name": "Vision patch merger spatial shuffle MLP",
"group": "Image and video",
"op": "vision patch merger: 2x2 shuffle + LayerNorm + GELU MLP",
"direction": "forward",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"workloads": 15
},
{
"board": "L1",
"id": "021_vision_cu_seqlens_variable_length_attention",
"name": "Vision cu seqlens variable length attention",
"group": "Image and video",
"op": "vision varlen attention (cu_seqlens) with QKV proj + RoPE",
"direction": "forward",
"model": "Qwen/Qwen3-VL-8B-Instruct",
"workloads": 16
},
{
"board": "L1",
"id": "022_vision_mlp_gelu_backward",
"name": "Vision MLP GELU backward",
"group": "Backward pass",
"op": "vision MLP GELU backward",
"direction": "backward",
"model": "Qwen/Qwen3-VL-8B-Instruct",
"workloads": 16
},
{
"board": "L1",
"id": "023_multimodal_rope_position_computation_with_grid_based_indexing",
"name": "Multimodal RoPE position computation with grid based indexing",
"group": "Image and video",
"op": "multimodal RoPE positions + learned position-embedding interpolation",
"direction": "forward",
"model": "Qwen/Qwen3-VL-235B-A22B-Instruct",
"workloads": 16
},
{
"board": "L1",
"id": "024_vision_rotary_position_embedding_generation_backward",
"name": "Vision rotary position embedding generation backward",
"group": "Backward pass",
"op": "vision RoPE generation backward",
"direction": "backward",
"model": "Qwen/Qwen-Image-Edit-2509",
"workloads": 16
},
{
"board": "L1",
"id": "025_video_latent_gelu_activation",
"name": "Video latent GELU activation",
"group": "Image and video",
"op": "GELU-tanh elementwise (video latents)",
"direction": "forward",
"model": "Wan-AI/Wan2.2-T2V-A14B",
"workloads": 16
},
{
"board": "L1",
"id": "026_video_patch_embedding_projection",
"name": "Video patch embedding projection",
"group": "Image and video",
"op": "Conv3D video patch embed + position embed + LayerNorm",
"direction": "forward",
"model": "Wan-AI/Wan2.2-T2V-A14B",
"workloads": 4
},
{
"board": "L1",
"id": "027_video_spatial_attention_with_rope_3d",
"name": "Video spatial attention with RoPE 3D",
"group": "Image and video",
"op": "video spatial attention with 3D RoPE",
"direction": "forward",
"model": "Wan-AI/Wan2.2-T2V-A14B",
"workloads": 16
},
{
"board": "L1",
"id": "028_hybrid_attention_mask_preparation",
"name": "Hybrid attention mask preparation",
"group": "Attention",
"op": "causal + sliding-window mask construction",
"direction": "forward",
"model": "XiaomiMiMo/MiMo-V2-Flash",
"workloads": 12
},
{
"board": "L1",
"id": "029_mamba_conv1d_with_gating",
"name": "Mamba Conv1D with gating",
"group": "State space and conv",
"op": "Mamba in-proj + causal depthwise conv1d + SiLU gate",
"direction": "forward",
"model": "ai21labs/AI21-Jamba-Large-1.7",
"workloads": 16
},
{
"board": "L1",
"id": "030_attention_output_projection_with_residual",
"name": "Attention output projection with residual",
"group": "Projections and GEMM",
"op": "o_proj GEMM (2560x2560) + residual add",
"direction": "forward",
"model": "ai21labs/AI21-Jamba-Reasoning-3B",
"workloads": 16
},
{
"board": "L1",
"id": "031_repeat_kv_attention_matmul",
"name": "Repeat KV attention matmul",
"group": "Attention",
"op": "GQA repeat_kv + Q@K^T scores",
"direction": "forward",
"model": "allenai/Olmo-3-7B-Think",
"workloads": 16
},
{
"board": "L1",
"id": "032_attention_weights_matmul_with_value_projection",
"name": "Attention weights matmul with value projection",
"group": "Attention",
"op": "attention weights @ V + transpose / reshape",
"direction": "forward",
"model": "allenai/Olmo-3-32B-Think",
"workloads": 16
},
{
"board": "L1",
"id": "033_post_norm_residual",
"name": "Post norm residual",
"group": "Normalization",
"op": "post-norm residual: residual + RMSNorm(x)",
"direction": "forward",
"model": "allenai/Olmo-3-1025-7B",
"workloads": 16
},
{
"board": "L1",
"id": "034_flux_multi_axis_rope_frequency_computation",
"name": "Flux multi axis RoPE frequency computation",
"group": "Image and video",
"op": "FLUX multi-axis RoPE frequencies",
"direction": "forward",
"model": "black-forest-labs/FLUX.1-Fill-dev",
"workloads": 16
},
{
"board": "L1",
"id": "035_flux_ada_layer_norm_zero_modulation_extraction",
"name": "Flux ada layer norm zero modulation extraction",
"group": "Image and video",
"op": "FLUX AdaLN-Zero modulation linear + chunk",
"direction": "forward",
"model": "black-forest-labs/FLUX.1-Kontext-dev",
"workloads": 16
},
{
"board": "L1",
"id": "036_flux_output_norm_projection_chain",
"name": "Flux output norm projection chain",
"group": "Image and video",
"op": "FLUX final AdaLN + output projection",
"direction": "forward",
"model": "black-forest-labs/FLUX.1-Kontext-dev",
"workloads": 16
},
{
"board": "L1",
"id": "037_flux_feedforward_gelu_approximate",
"name": "Flux feedforward GELU approximate",
"group": "Image and video",
"op": "FLUX feed-forward with GELU-tanh",
"direction": "forward",
"model": "black-forest-labs/FLUX.1-dev",
"workloads": 16
},
{
"board": "L1",
"id": "038_flux_multi_head_rmsnorm_qk",
"name": "Flux multi head RMSNorm QK",
"group": "Image and video",
"op": "FLUX.2 per-head QK RMSNorm",
"direction": "forward",
"model": "black-forest-labs/FLUX.2-dev",
"workloads": 16
},
{
"board": "L1",
"id": "039_adaptive_instance_normalization_2d_backward",
"name": "Adaptive instance normalization 2D backward",
"group": "Backward pass",
"op": "adaptive instance norm 2D backward",
"direction": "backward",
"model": "chuxiaojie/NAFNet",
"workloads": 16
},
{
"board": "L1",
"id": "040_conv2d_residual_block",
"name": "Conv2D residual block",
"group": "Image and video",
"op": "NAFNet conv residual block",
"direction": "forward",
"model": "chuxiaojie/NAFNet",
"workloads": 16
},
{
"board": "L1",
"id": "041_strided_conv2d_downsampling",
"name": "Strided Conv2D downsampling",
"group": "Image and video",
"op": "NAFNet strided conv2d downsampling",
"direction": "forward",
"model": "chuxiaojie/NAFNet",
"workloads": 16
},
{
"board": "L1",
"id": "042_moe_expert_load_balancing_and_token_capacity_backward",
"name": "MoE expert load balancing and token capacity backward",
"group": "Backward pass",
"op": "MoE load-balancing loss / capacity backward",
"direction": "backward",
"model": "deepseek-ai/DeepSeek-R1-0528",
"workloads": 16
},
{
"board": "L1",
"id": "043_mla_fused_qkv_rope_split",
"name": "MLA fused QKV RoPE split",
"group": "Projections and GEMM",
"op": "MLA latent projections: q_a -> RMSNorm -> q_b, kv_a + split",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-V3",
"workloads": 16
},
{
"board": "L1",
"id": "044_moe_expert_computation",
"name": "MoE expert computation",
"group": "MLP and MoE",
"op": "MoE experts: SwiGLU per expert from given ids / weights",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-V3",
"workloads": 16
},
{
"board": "L1",
"id": "045_fused_linear_gelu_grn_linear",
"name": "Fused linear GELU GRN linear",
"group": "Image and video",
"op": "ConvNeXtV2 MLP + GRN",
"direction": "forward",
"model": "facebook/convnextv2-base-22k-224",
"workloads": 14
},
{
"board": "L1",
"id": "046_attention_softmax_with_softcapping_and_dropout",
"name": "Attention softmax with softcapping and dropout",
"group": "Attention",
"op": "attention softmax with logit soft-cap + dropout",
"direction": "forward",
"model": "google/gemma-3-1b-it",
"workloads": 16
},
{
"board": "L1",
"id": "047_attention_with_qk_norm_and_rope",
"name": "Attention with QK norm and RoPE",
"group": "Attention",
"op": "Gemma3 attention block: QKV/O proj, QK-RMSNorm, RoPE, soft-cap, causal",
"direction": "forward",
"model": "google/gemma-3-12b-it",
"workloads": 16
},
{
"board": "L1",
"id": "048_fused_gate_up_projection_with_swiglu",
"name": "Fused gate up projection with SwiGLU",
"group": "MLP and MoE",
"op": "fused gate+up GEMM with GELU-tanh gating (no down proj)",
"direction": "forward",
"model": "google/gemma-3-12b-it",
"workloads": 16
},
{
"board": "L1",
"id": "049_attention_qk_matmul_with_gqa_repeat_and_scaling",
"name": "Attention QK matmul with GQA repeat and scaling",
"group": "Attention",
"op": "GQA repeat + Q@K^T + scale",
"direction": "forward",
"model": "google/gemma-3-270m",
"workloads": 16
},
{
"board": "L1",
"id": "050_attention_qkv_projection_with_bias_and_reshape",
"name": "Attention QKV projection with bias and reshape",
"group": "Projections and GEMM",
"op": "QKV projection with bias + reshape",
"direction": "forward",
"model": "google/gemma-3-270m",
"workloads": 16
},
{
"board": "L1",
"id": "051_attention_qkv_with_qk_norm_single_kernel_backward",
"name": "Attention QKV with QK norm single kernel backward",
"group": "Backward pass",
"op": "QKV + QK-norm backward",
"direction": "backward",
"model": "google/gemma-3-270m",
"workloads": 16
},
{
"board": "L1",
"id": "052_altup_hidden_state_collapse_with_magnitude_normalization",
"name": "AltUp hidden state collapse with magnitude normalization",
"group": "Other",
"op": "Gemma3n AltUp stream collapse",
"direction": "forward",
"model": "google/gemma-3n-E2B",
"workloads": 16
},
{
"board": "L1",
"id": "053_gaussian_topk_sparse_activation",
"name": "Gaussian topk sparse activation",
"group": "Other",
"op": "Gemma3n Gaussian top-k sparse activation",
"direction": "forward",
"model": "google/gemma-3n-E2B",
"workloads": 12
},
{
"board": "L1",
"id": "054_audio_attention_qkv_projection_with_normalization",
"name": "Audio attention QKV projection with normalization",
"group": "Audio",
"op": "audio attention QKV proj + per-head RMSNorm",
"direction": "forward",
"model": "google/gemma-3n-E4B",
"workloads": 16
},
{
"board": "L1",
"id": "055_attention_qkv_projection_with_gqa_reshape",
"name": "Attention QKV projection with GQA reshape",
"group": "Projections and GEMM",
"op": "QKV projection + GQA reshape",
"direction": "forward",
"model": "ibm-granite/granite-4.0-1b",
"workloads": 16
},
{
"board": "L1",
"id": "056_mamba_ssm_dt_projection_softplus_clamp_backward",
"name": "Mamba SSM dt projection softplus clamp backward",
"group": "Backward pass",
"op": "Mamba dt projection + softplus + clamp backward",
"direction": "backward",
"model": "ibm-granite/granite-4.0-micro",
"workloads": 16
},
{
"board": "L1",
"id": "057_mtp_shifted_embedding_with_dual_rms_norm_fusion",
"name": "MTP shifted embedding with dual RMS norm fusion",
"group": "Full decoder layers",
"op": "MTP input: shifted embedding + dual RMSNorm + concat + proj",
"direction": "forward",
"model": "inclusionAI/Ling-flash-2.0",
"workloads": 16
},
{
"board": "L1",
"id": "058_moe_expert_token_radix_sort_with_prefix_sum",
"name": "MoE expert token radix sort with prefix sum",
"group": "MLP and MoE",
"op": "MoE token sort by expert (counting sort + prefix sum)",
"direction": "forward",
"model": "inclusionAI/Ling-mini-2.0",
"workloads": 16
},
{
"board": "L1",
"id": "059_moe_group_score_aggregation_and_masking",
"name": "MoE group score aggregation and masking",
"group": "MLP and MoE",
"op": "MoE group-limited routing step: top-2 group sums, top-4 groups, mask",
"direction": "forward",
"model": "inclusionAI/Ling-mini-2.0",
"workloads": 16
},
{
"board": "L1",
"id": "060_fused_attention_qk_matmul_scale_mask_softmax_backward",
"name": "Fused attention QK matmul scale mask softmax backward",
"group": "Backward pass",
"op": "attention QK scale mask softmax backward",
"direction": "backward",
"model": "meta-llama/Llama-3.1-405B",
"workloads": 16
},
{
"board": "L1",
"id": "061_tanh_gated_residual_add_backward",
"name": "Tanh gated residual add backward",
"group": "Backward pass",
"op": "tanh-gated residual add backward",
"direction": "backward",
"model": "meta-llama/Llama-3.2-11B-Vision",
"workloads": 16
},
{
"board": "L1",
"id": "062_kv_cache_update_with_rope_backward",
"name": "KV cache update with RoPE backward",
"group": "Backward pass",
"op": "KV-cache update with RoPE backward",
"direction": "backward",
"model": "mistralai/Devstral-Small-2507",
"workloads": 16
},
{
"board": "L1",
"id": "063_attention_output_reshape_and_projection",
"name": "Attention output reshape and projection",
"group": "Projections and GEMM",
"op": "attention output reshape + o_proj (heads*v_head_dim -> hidden)",
"direction": "forward",
"model": "moonshotai/Kimi-K2-Instruct-0905",
"workloads": 16
},
{
"board": "L1",
"id": "064_latent_kv_expansion_with_split",
"name": "Latent KV expansion with split",
"group": "Projections and GEMM",
"op": "MLA latent KV expansion: RMSNorm + kv_b_proj + split K_nope / V",
"direction": "forward",
"model": "moonshotai/Kimi-K2-Instruct-0905",
"workloads": 16
},
{
"board": "L1",
"id": "065_fused_statistics_projection_and_split",
"name": "Fused statistics projection and split",
"group": "Audio",
"op": "Conv1d stats projection + split (TTS)",
"direction": "forward",
"model": "myshell-ai/OpenVoice",
"workloads": 16
},
{
"board": "L1",
"id": "066_masked_softmax_with_attention_dropout_backward",
"name": "Masked softmax with attention dropout backward",
"group": "Backward pass",
"op": "masked softmax + dropout backward",
"direction": "backward",
"model": "myshell-ai/OpenVoice",
"workloads": 16
},
{
"board": "L1",
"id": "067_flash_attention_gqa_ultralong",
"name": "Flash attention GQA ultralong",
"group": "Attention",
"op": "GQA attention block: QKV proj, RoPE, causal SDPA, o_proj (no QK-norm)",
"direction": "forward",
"model": "nvidia/Llama-3.1-Nemotron-8B-UltraLong-1M-Instruct",
"workloads": 18
},
{
"board": "L1",
"id": "068_fused_rope_application_backward",
"name": "Fused RoPE application backward",
"group": "Backward pass",
"op": "RoPE application backward",
"direction": "backward",
"model": "nvidia/Llama-3.1-Nemotron-8B-UltraLong-1M-Instruct",
"workloads": 16
},
{
"board": "L1",
"id": "069_rms_norm",
"name": "RMS norm",
"group": "Normalization",
"op": "residual add + RMSNorm (returns the normed output only)",
"direction": "forward",
"model": "nvidia/Llama-3.3-Nemotron-70B-Select",
"workloads": 16
},
{
"board": "L1",
"id": "070_mamba2_fused_intra_chunk_diagonal_computation",
"name": "Mamba2 fused intra chunk diagonal computation",
"group": "State space and conv",
"op": "Mamba2 SSD intra-chunk diagonal block",
"direction": "forward",
"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
"workloads": 14
},
{
"board": "L1",
"id": "071_kv_cache_update_with_rope",
"name": "KV cache update with RoPE",
"group": "Attention",
"op": "KV-cache update with RoPE on K",
"direction": "forward",
"model": "nvidia/Phi-4-reasoning-plus-NVFP4",
"workloads": 16
},
{
"board": "L1",
"id": "072_cross_attention_qkv_projection_with_gqa_repeat_backward",
"name": "Cross attention QKV projection with GQA repeat backward",
"group": "Backward pass",
"op": "cross-attention QKV + GQA repeat backward",
"direction": "backward",
"model": "nvidia/parakeet-tdt-0.6b-v3",
"workloads": 16
},
{
"board": "L1",
"id": "073_fused_encoder_final_norm_to_decoder_cross_attention_kv_projection",
"name": "Fused encoder final norm to decoder cross attention KV projection",
"group": "Audio",
"op": "encoder final RMSNorm + cross-attention K/V projection",
"direction": "forward",
"model": "nvidia/parakeet-tdt-0.6b-v3",
"workloads": 16
},
{
"board": "L1",
"id": "074_fused_gated_mlp_silu",
"name": "Fused gated MLP SiLU",
"group": "MLP and MoE",
"op": "gated MLP: fused gate_up GEMM + SiLU*up + down",
"direction": "forward",
"model": "nvidia/parakeet-tdt-0.6b-v3",
"workloads": 16
},
{
"board": "L1",
"id": "075_grouped_query_self_attention_with_rope",
"name": "Grouped query self attention with RoPE",
"group": "Audio",
"op": "GQA self-attention block with RoPE (encoder / decoder)",
"direction": "forward",
"model": "nvidia/parakeet-tdt-0.6b-v3",
"workloads": 16
},
{
"board": "L1",
"id": "076_batched_expert_forward",
"name": "Batched expert forward",
"group": "MLP and MoE",
"op": "gpt-oss experts: gate_up + clamped SwiGLU (alpha, limit) + down, dense over all experts",
"direction": "forward",
"model": "openai/gpt-oss-120b",
"workloads": 14
},
{
"board": "L1",
"id": "077_whisper_decoder_output_projection",
"name": "Whisper decoder output projection",
"group": "Projections and GEMM",
"op": "Whisper LM head GEMM",
"direction": "forward",
"model": "openai/whisper-large-v3",
"workloads": 16
},
{
"board": "L1",
"id": "078_group_norm_fusion",
"name": "Group norm fusion",
"group": "Image and video",
"op": "GroupNorm (SD UNet)",
"direction": "forward",
"model": "runwayml/stable-diffusion-v1-5",
"workloads": 16
},
{
"board": "L1",
"id": "079_ImageNet_83.6_ssm_output_projection_gate_multiply_backward",
"name": "ImageNet 83.6 SSM output projection gate multiply backward",
"group": "Backward pass",
"op": "SSM output projection gate multiply backward",
"direction": "backward",
"model": "saurabhati/VMamba",
"workloads": 16
},
{
"board": "L1",
"id": "080_adaptive_layernorm_continuous_with_modulation",
"name": "Adaptive LayerNorm continuous with modulation",
"group": "Image and video",
"op": "SD3.5 AdaLayerNormContinuous",
"direction": "forward",
"model": "stabilityai/stable-diffusion-3.5-large",
"workloads": 16
},
{
"board": "L1",
"id": "081_joint_attention_context_projection",
"name": "Joint attention context projection",
"group": "Image and video",
"op": "SD3.5 joint-attention context projection",
"direction": "forward",
"model": "stabilityai/stable-diffusion-3.5-large",
"workloads": 16
},
{
"board": "L1",
"id": "082_qk_norm_scaled_dot_product_attention",
"name": "QK norm scaled dot product attention",
"group": "Image and video",
"op": "SD3.5 QK-LayerNorm SDPA",
"direction": "forward",
"model": "stabilityai/stable-diffusion-3.5-medium",
"workloads": 16
},
{
"board": "L1",
"id": "083_attention_score_value_matmul",
"name": "Attention score value matmul",
"group": "Image and video",
"op": "SDXL attention weights @ V",
"direction": "forward",
"model": "stabilityai/stable-diffusion-xl-base-1.0",
"workloads": 16
},
{
"board": "L1",
"id": "084_silu_activation_backward",
"name": "SiLU activation backward",
"group": "Backward pass",
"op": "SiLU backward",
"direction": "backward",
"model": "stabilityai/stable-diffusion-xl-base-1.0",
"workloads": 16
},
{
"board": "L1",
"id": "085_geglu_activation",
"name": "GEGLU activation",
"group": "Image and video",
"op": "GEGLU activation",
"direction": "forward",
"model": "stabilityai/stable-diffusion-xl-refiner-1.0",
"workloads": 16
},
{
"board": "L1",
"id": "086_sam_hq_mask_decoder_iou_hypernetwork_fusion",
"name": "SAM hq mask decoder IoU hypernetwork fusion",
"group": "Image and video",
"op": "SAM-HQ IoU head + hypernetwork MLPs",
"direction": "forward",
"model": "syscv-community/sam-hq-vit-base",
"workloads": 16
},
{
"board": "L1",
"id": "087_embedding_with_initial_layernorm_backward",
"name": "Embedding with initial LayerNorm backward",
"group": "Backward pass",
"op": "embedding + LayerNorm backward",
"direction": "backward",
"model": "taozi555/jamba-mini-1.7-rp",
"workloads": 16
},
{
"board": "L1",
"id": "088_rotary_position_embedding_application",
"name": "Rotary position embedding application",
"group": "Image and video",
"op": "RoPE application to Q/K (HunyuanImage)",
"direction": "forward",
"model": "tencent/HunyuanImage-2.1",
"workloads": 16
},
{
"board": "L1",
"id": "089_vae_attention_block_with_groupnorm",
"name": "VAE attention block with GroupNorm",
"group": "Image and video",
"op": "VAE attention block with GroupNorm",
"direction": "forward",
"model": "tencent/HunyuanImage-2.1",
"workloads": 16
},
{
"board": "L1",
"id": "090_batched_2d_rope_position_encoding_backward",
"name": "Batched 2D RoPE position encoding backward",
"group": "Backward pass",
"op": "batched 2D RoPE backward",
"direction": "backward",
"model": "tencent/HunyuanImage-3.0",
"workloads": 16
},
{
"board": "L1",
"id": "091_fused_gqa_qkv_rope_qk_norm_backward",
"name": "Fused GQA QKV RoPE QK norm backward",
"group": "Backward pass",
"op": "fused GQA QKV + RoPE + QK-norm backward",
"direction": "backward",
"model": "tencent/HunyuanImage-3.0",
"workloads": 16
},
{
"board": "L1",
"id": "092_gqa_attention_with_qk_norm",
"name": "GQA attention with QK norm",
"group": "Attention",
"op": "GQA attention block: QKV proj (bias), QK-RMSNorm, RoPE, causal, o_proj",
"direction": "forward",
"model": "zai-org/GLM-4.5-Air",
"workloads": 16
},
{
"board": "L1",
"id": "093_grouped_topk_moe_routing_backward",
"name": "Grouped topk MoE routing backward",
"group": "Backward pass",
"op": "grouped top-k MoE routing backward",
"direction": "backward",
"model": "zai-org/GLM-4.7",
"workloads": 16
},
{
"board": "L1",
"id": "094_time_decay_exponential_stabilization",
"name": "Time decay exponential stabilization",
"group": "Other",
"op": "RWKV time-decay exponent with stabilisation",
"direction": "forward",
"model": "zhiyuan8/RWKV-v7-1.5B-G1-GGUF",
"workloads": 16
},
{
"board": "L2",
"id": "001_fused_vision_multihead_attention_with_norms_backward",
"name": "Fused vision multihead attention with norms backward",
"group": "Backward pass",
"op": "vision MHA with norms backward",
"direction": "backward",
"model": "1024m/Molmo-7B-D-Base",
"workloads": 16
},
{
"board": "L2",
"id": "002_decoder_layer_full_block",
"name": "Decoder layer full block",
"group": "Full decoder layers",
"op": "Llama decoder layer: RMSNorm, GQA + RoPE, residual, RMSNorm, SwiGLU MLP, residual",
"direction": "forward",
"model": "NousResearch/DeepHermes-3-Llama-3-8B-Preview",
"workloads": 18
},
{
"board": "L2",
"id": "003_grouped_query_attention_with_rope_backward",
"name": "Grouped query attention with RoPE backward",
"group": "Backward pass",
"op": "GQA attention with RoPE backward",
"direction": "backward",
"model": "NousResearch/DeepHermes-3-Mistral-24B-Preview",
"workloads": 16
},
{
"board": "L2",
"id": "004_fused_residual_rms_mlp",
"name": "Fused residual RMS MLP",
"group": "MLP and MoE",
"op": "residual + RMSNorm + SwiGLU MLP",
"direction": "forward",
"model": "NousResearch/Hermes-4-405B",
"workloads": 16
},
{
"board": "L2",
"id": "005_swiglu_mlp_backward",
"name": "SwiGLU MLP backward",
"group": "Backward pass",
"op": "SwiGLU MLP backward",
"direction": "backward",
"model": "NousResearch/Hermes-4-405B",
"workloads": 18
},
{
"board": "L2",
"id": "006_multimodal_rope_position_calculation",
"name": "Multimodal RoPE position calculation",
"group": "Attention",
"op": "multimodal 3D RoPE position-id calculation",
"direction": "forward",
"model": "Qwen/Qwen2.5-VL-7B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "007_multimodal_rotary_embedding_attention",
"name": "Multimodal rotary embedding attention",
"group": "Attention",
"op": "GQA attention with multimodal 3D RoPE",
"direction": "forward",
"model": "Qwen/Qwen2.5-VL-7B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "008_moe_sparse_routing_and_dispatch",
"name": "MoE sparse routing and dispatch",
"group": "MLP and MoE",
"op": "MoE block: softmax router + top-8 + SwiGLU experts + weighted sum",
"direction": "forward",
"model": "Qwen/Qwen3-30B-A3B-Instruct-2507",
"workloads": 16
},
{
"board": "L2",
"id": "009_decoder_layer_with_residual_connections",
"name": "Decoder layer with residual connections",
"group": "Full decoder layers",
"op": "MoE decoder layer: RMSNorm, GQA + QK-norm, MoE, residuals",
"direction": "forward",
"model": "Qwen/Qwen3-Coder-30B-A3B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "010_moe_expert_computation_with_weighted_accumulation",
"name": "MoE expert computation with weighted accumulation",
"group": "MLP and MoE",
"op": "MoE experts SwiGLU + weighted accumulation (given top-k)",
"direction": "forward",
"model": "Qwen/Qwen3-Coder-30B-A3B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "011_moe_sparse_routing_and_dispatch_backward",
"name": "MoE sparse routing and dispatch backward",
"group": "Backward pass",
"op": "MoE routing + dispatch backward",
"direction": "backward",
"model": "Qwen/Qwen3-Coder-30B-A3B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "012_moe_expert_batched_execution_with_capacity_factor",
"name": "MoE expert batched execution with capacity factor",
"group": "MLP and MoE",
"op": "MoE experts with capacity-factor batching",
"direction": "forward",
"model": "Qwen/Qwen3-Coder-480B-A35B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "013_expert_weighted_aggregation_with_shared_expert",
"name": "Expert weighted aggregation with shared expert",
"group": "MLP and MoE",
"op": "MoE experts + weighted sum + sigmoid-gated shared expert",
"direction": "forward",
"model": "Qwen/Qwen3-Next-80B-A3B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "014_audio_encoder_varlen_attention_with_chunking_backward",
"name": "Audio encoder varlen attention with chunking backward",
"group": "Backward pass",
"op": "audio varlen attention with chunking backward",
"direction": "backward",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "015_audio_sinusoidal_position_embedding_with_conv_projection",
"name": "Audio sinusoidal position embedding with conv projection",
"group": "Audio",
"op": "audio sinusoidal position embed + Conv2d downsampling + proj",
"direction": "forward",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "016_moe_expert_mlp_with_load_balancing",
"name": "MoE expert MLP with load balancing",
"group": "MLP and MoE",
"op": "MoE block: softmax router + top-8 + SwiGLU experts (returns router logits)",
"direction": "forward",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "017_fused_vision_cu_seqlens_attention_with_2d_rope_backward",
"name": "Fused vision cu seqlens attention with 2D RoPE backward",
"group": "Backward pass",
"op": "vision varlen attention with 2D RoPE backward",
"direction": "backward",
"model": "Qwen/Qwen3-VL-4B-Instruct",
"workloads": 16
},
{
"board": "L2",
"id": "018_cu_seqlens_variable_length_vision_attention",
"name": "Cu seqlens variable length vision attention",
"group": "Image and video",
"op": "vision varlen attention (cu_seqlens) with RoPE",
"direction": "forward",
"model": "Qwen/Qwen-Image-Edit-2509",
"workloads": 16
},
{
"board": "L2",
"id": "019_decoder_layer_fused_attention_mlp",
"name": "Decoder layer fused attention MLP",
"group": "Full decoder layers",
"op": "Qwen2-VL decoder layer (M-RoPE GQA + SwiGLU)",
"direction": "forward",
"model": "Qwen/Qwen-Image-Edit-2509",
"workloads": 16
},
{
"board": "L2",
"id": "020_decoder_layer_pre_post_norm_residual",
"name": "Decoder layer pre post norm residual",
"group": "Full decoder layers",
"op": "decoder layer with pre / post norm residuals",
"direction": "forward",
"model": "RekaAI/reka-flash-3",
"workloads": 16
},
{
"board": "L2",
"id": "021_cross_attention_text_video_conditioning_backward",
"name": "Cross attention text video conditioning backward",
"group": "Backward pass",
"op": "text-video cross-attention backward",
"direction": "backward",
"model": "Wan-AI/Wan2.2-T2V-A14B",
"workloads": 16
},
{
"board": "L2",
"id": "022_video_latent_denoising_unet_block",
"name": "Video latent denoising UNet block",
"group": "Image and video",
"op": "video latent denoising block",
"direction": "forward",
"model": "Wan-AI/Wan2.2-T2V-A14B",
"workloads": 16
},
{
"board": "L2",
"id": "023_video_latent_vae_encoder_downsampling",
"name": "Video latent VAE encoder downsampling",
"group": "Image and video",
"op": "video VAE encoder downsampling",
"direction": "forward",
"model": "Wan-AI/Wan2.2-T2V-A14B",
"workloads": 16
},
{
"board": "L2",
"id": "024_moe_expert_parallel_execution",
"name": "MoE expert parallel execution",
"group": "MLP and MoE",
"op": "MoE experts (256, given top-k) SwiGLU + scatter-add",
"direction": "forward",
"model": "XiaomiMiMo/MiMo-V2-Flash",
"workloads": 16
},
{
"board": "L2",
"id": "025_moe_expert_parallel_execution_backward",
"name": "MoE expert parallel execution backward",
"group": "Backward pass",
"op": "MoE experts backward",
"direction": "backward",
"model": "XiaomiMiMo/MiMo-V2-Flash",
"workloads": 16
},
{
"board": "L2",
"id": "026_moe_expert_parallel_execution_with_weighted_aggregation",
"name": "MoE expert parallel execution with weighted aggregation",
"group": "MLP and MoE",
"op": "MoE experts (16, top-2) with weighted aggregation",
"direction": "forward",
"model": "ai21labs/AI21-Jamba-Large-1.7",
"workloads": 16
},
{
"board": "L2",
"id": "027_grouped_query_attention_with_yarn_rope_and_qk_norm",
"name": "Grouped query attention with YaRN RoPE and QK norm",
"group": "Attention",
"op": "GQA attention with YaRN RoPE, QK-norm, sliding window",
"direction": "forward",
"model": "allenai/Olmo-3-32B-Think",
"workloads": 16
},
{
"board": "L2",
"id": "028_gqa_rotary_attention_core_backward",
"name": "GQA rotary attention core backward",
"group": "Backward pass",
"op": "GQA rotary attention core backward",
"direction": "backward",
"model": "baidu/ERNIE-4.5-300B-A47B-PT",
"workloads": 16
},
{
"board": "L2",
"id": "029_moe_sparse_routing_and_dispatch",
"name": "MoE sparse routing and dispatch",
"group": "MLP and MoE",
"op": "MoE block: biased router, top-k, experts + shared expert",
"direction": "forward",
"model": "baidu/ERNIE-4.5-300B-A47B-PT",
"workloads": 9
},
{
"board": "L2",
"id": "030_flux_concatenated_sequence_processing_with_split",
"name": "Flux concatenated sequence processing with split",
"group": "Image and video",
"op": "FLUX concat text+image, linear, split",
"direction": "forward",
"model": "black-forest-labs/FLUX.1-Kontext-dev",
"workloads": 16
},
{
"board": "L2",
"id": "031_flux_timestep_guidance_projection_embedding",
"name": "Flux timestep guidance projection embedding",
"group": "Image and video",
"op": "FLUX timestep + guidance embedding",
"direction": "forward",
"model": "black-forest-labs/FLUX.1-dev",
"workloads": 16
},
{
"board": "L2",
"id": "032_dual_stream_attention_with_conditional_cross_attention",
"name": "Dual stream attention with conditional cross attention",
"group": "Image and video",
"op": "FIBO dual-stream attention block",
"direction": "forward",
"model": "briaai/FIBO",
"workloads": 16
},
{
"board": "L2",
"id": "033_multi_scale_feature_pyramid",
"name": "Multi scale feature pyramid",
"group": "Image and video",
"op": "multi-scale feature pyramid (U-Net)",
"direction": "forward",
"model": "chuxiaojie/NAFNet",
"workloads": 16
},
{
"board": "L2",
"id": "034_vision_language_cross_attention_fusion",
"name": "Vision language cross attention fusion",
"group": "Attention",
"op": "language-to-vision cross-attention with 1D / 3D RoPE",
"direction": "forward",
"model": "cybermotaz/Qwen3-VL-32B-Instruct-NVFP4",
"workloads": 16
},
{
"board": "L2",
"id": "035_convnextv2_block_with_grn",
"name": "ConvNeXtV2 block with GRN",
"group": "Image and video",
"op": "ConvNeXtV2 block with GRN",
"direction": "forward",
"model": "facebook/convnextv2-base-22k-224",
"workloads": 16
},
{
"board": "L2",
"id": "036_convnextv2_layer_with_nhwc_persistence_backward",
"name": "ConvNeXtV2 layer with NHWC persistence backward",
"group": "Backward pass",
"op": "ConvNeXtV2 layer backward",
"direction": "backward",
"model": "facebook/convnextv2-base-22k-224",
"workloads": 14
},
{
"board": "L2",
"id": "037_qk_norm_rope_attention_core_backward",
"name": "QK norm RoPE attention core backward",
"group": "Backward pass",
"op": "QK-norm RoPE attention core backward",
"direction": "backward",
"model": "google/gemma-3-4b-it",
"workloads": 16
},
{
"board": "L2",
"id": "038_audio_relative_position_attention",
"name": "Audio relative position attention",
"group": "Audio",
"op": "Gemma3n audio relative-position attention",
"direction": "forward",
"model": "google/gemma-3n-E2B",
"workloads": 14
},
{
"board": "L2",
"id": "039_kv_shared_attention_with_rope",
"name": "KV shared attention with RoPE",
"group": "Attention",
"op": "Gemma3n KV-shared attention with RoPE",
"direction": "forward",
"model": "google/gemma-3n-E2B",
"workloads": 16
},
{
"board": "L2",
"id": "040_altup_predict_correction_cycle_backward",
"name": "AltUp predict correction cycle backward",
"group": "Backward pass",
"op": "AltUp predict / correct backward",
"direction": "backward",
"model": "google/gemma-3n-E4B",
"workloads": 16
},
{
"board": "L2",
"id": "041_kv_shared_attention_with_dual_rope",
"name": "KV shared attention with dual RoPE",
"group": "Attention",
"op": "Gemma3n KV-shared attention with dual RoPE",
"direction": "forward",
"model": "google/gemma-3n-E4B",
"workloads": 16
},
{
"board": "L2",
"id": "042_ffn_gelu_projection_fused_backward",
"name": "FFN GELU projection fused backward",
"group": "Backward pass",
"op": "FFN GELU projection backward",
"direction": "backward",
"model": "hexgrad/Kokoro-82M",
"workloads": 16
},
{
"board": "L2",
"id": "043_mamba_chunk_scan_with_segsum",
"name": "Mamba chunk scan with segsum",
"group": "State space and conv",
"op": "Mamba2 chunk scan with segsum",
"direction": "forward",
"model": "ibm-granite/granite-4.0-1b",
"workloads": 16
},
{
"board": "L2",
"id": "044_mamba_discretization_and_segsum",
"name": "Mamba discretization and segsum",
"group": "State space and conv",
"op": "Mamba SSM discretisation + segsum",
"direction": "forward",
"model": "ibm-granite/granite-4.0-micro",
"workloads": 16
},
{
"board": "L2",
"id": "045_audio_encoder_to_language_model_multimodal_fusion",
"name": "Audio encoder to language model multimodal fusion",
"group": "Audio",
"op": "speech encoder + Q-Former + scatter into LM embeddings",
"direction": "forward",
"model": "ibm-granite/granite-speech-3.3-8b",
"workloads": 16
},
{
"board": "L2",
"id": "046_conformer_relative_position_attention",
"name": "Conformer relative position attention",
"group": "Audio",
"op": "Conformer relative-position attention",
"direction": "forward",
"model": "ibm-granite/granite-speech-3.3-8b",
"workloads": 16
},
{
"board": "L2",
"id": "047_moe_training_token_repeat_and_expert_computation",
"name": "MoE training token repeat and expert computation",
"group": "MLP and MoE",
"op": "MoE training forward: token repeat + masked expert SwiGLU",
"direction": "forward",
"model": "inclusionAI/Ling-1T",
"workloads": 16
},
{
"board": "L2",
"id": "048_moe_expert_inference_batched_dispatch",
"name": "MoE expert inference batched dispatch",
"group": "MLP and MoE",
"op": "MoE experts inference dispatch (sorted by expert)",
"direction": "forward",
"model": "inclusionAI/Ling-flash-2.0",
"workloads": 16
},
{
"board": "L2",
"id": "049_group_limited_topk_routing",
"name": "Group limited topk routing",
"group": "MLP and MoE",
"op": "group-limited top-k routing: router GEMM, sigmoid + bias, 8 groups / top-4 / top-8",
"direction": "forward",
"model": "inclusionAI/Ring-flash-2.0",
"workloads": 16
},
{
"board": "L2",
"id": "050_vae_decoder_mid_block_attention_resnet",
"name": "VAE decoder mid block attention ResNet",
"group": "Image and video",
"op": "VAE decoder mid-block (ResNet + attention)",
"direction": "forward",
"model": "kaiyuyue/FLUX.2-dev-vae",
"workloads": 11
},
{
"board": "L2",
"id": "051_seqlen-finetuned-reconstructed_hyena_complete_forward_block",
"name": "Seqlen-finetuned-reconstructed Hyena complete forward block",
"group": "State space and conv",
"op": "Hyena block forward",
"direction": "forward",
"model": "magicslabnu/Hyena-small-32k",
"workloads": 16
},
{
"board": "L2",
"id": "052_seqlen-finetuned-reconstructed_hyena_gating_and_output_projection",
"name": "Seqlen-finetuned-reconstructed Hyena gating and output projection",
"group": "State space and conv",
"op": "Hyena gating + FFT conv + output projection",
"direction": "forward",
"model": "magicslabnu/Hyena-small-32k",
"workloads": 16
},
{
"board": "L2",
"id": "053_text_decoder_layer_with_self_attention_and_mlp",
"name": "Text decoder layer with self attention and MLP",
"group": "Full decoder layers",
"op": "Mllama text decoder layer: RMSNorm, GQA + RoPE, SwiGLU MLP, residuals",
"direction": "forward",
"model": "meta-llama/Llama-3.2-11B-Vision",
"workloads": 16
},
{
"board": "L2",
"id": "054_vision_encoder_layer_with_gated_residuals",
"name": "Vision encoder layer with gated residuals",
"group": "Image and video",
"op": "vision encoder layer with tanh-gated residuals",
"direction": "forward",
"model": "meta-llama/Llama-3.2-11B-Vision",
"workloads": 16
},
{
"board": "L2",
"id": "055_audio_encoder_conv_positional_layer_stack",
"name": "Audio encoder conv positional layer stack",
"group": "Audio",
"op": "audio conv stem + first encoder layer",
"direction": "forward",
"model": "mistralai/Voxtral-Small-24B-2507",
"workloads": 15
},
{
"board": "L2",
"id": "056_language_model_decoder_prenorm_attention_ffn_residual_backward",
"name": "Language model decoder prenorm attention FFN residual backward",
"group": "Backward pass",
"op": "decoder layer backward",
"direction": "backward",
"model": "mistralai/Voxtral-Small-24B-2507",
"workloads": 16
},
{
"board": "L2",
"id": "057_residual_coupling_flow_block",
"name": "Residual coupling flow block",
"group": "Audio",
"op": "normalising-flow residual coupling block (TTS)",
"direction": "forward",
"model": "myshell-ai/OpenVoice",
"workloads": 16
},
{
"board": "L2",
"id": "058_mamba2_selective_scan",
"name": "Mamba2 selective scan",
"group": "State space and conv",
"op": "Mamba2 selective scan with conv + gated norm",
"direction": "forward",
"model": "nvidia/Nemotron-H-47B-Base-8K",
"workloads": 16
},
{
"board": "L2",
"id": "059_decoder_layer_full_block",
"name": "Decoder layer full block",
"group": "Full decoder layers",
"op": "Qwen3 decoder layer: RMSNorm, QKV + QK-norm, RoPE, GQA, o_proj, SwiGLU MLP",
"direction": "forward",
"model": "nvidia/Qwen3-14B-NVFP4",
"workloads": 16
},
{
"board": "L2",
"id": "060_chunk_gated_delta_rule_linear_attention",
"name": "Chunk gated delta rule linear attention",
"group": "Attention",
"op": "chunked gated delta-rule linear attention",
"direction": "forward",
"model": "nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4",
"workloads": 16
},
{
"board": "L2",
"id": "061_sparse_moe_routing_and_experts",
"name": "Sparse MoE routing and experts",
"group": "MLP and MoE",
"op": "MoE block: softmax router top-10 + experts + sigmoid-gated shared expert",
"direction": "forward",
"model": "nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4",
"workloads": 16
},
{
"board": "L2",
"id": "062_decoder_complete_layer",
"name": "Decoder complete layer",
"group": "Audio",
"op": "speech decoder layer: self-attn + cross-attn + MLP",
"direction": "forward",
"model": "nvidia/canary-qwen-2.5b",
"workloads": 16
},
{
"board": "L2",
"id": "063_encoder_layer_dual_residual_norm_chain",
"name": "Encoder layer dual residual norm chain",
"group": "Audio",
"op": "speech encoder layer (dual residual-norm chain)",
"direction": "forward",
"model": "nvidia/parakeet-tdt-0.6b-v3",
"workloads": 16
},
{
"board": "L2",
"id": "064_multi_head_qkv_projection_with_rope_backward",
"name": "Multi head QKV projection with RoPE backward",
"group": "Backward pass",
"op": "QKV projection with RoPE backward",
"direction": "backward",
"model": "nvidia/parakeet-tdt-0.6b-v3",
"workloads": 16
},
{
"board": "L2",
"id": "065_sparse_expert_dispatch_and_combine",
"name": "Sparse expert dispatch and combine",
"group": "MLP and MoE",
"op": "gpt-oss sparse experts: gate_up + clamped SwiGLU + down + index_add",
"direction": "forward",
"model": "openai/gpt-oss-120b",
"workloads": 16
},
{
"board": "L2",
"id": "066_resnet_block_with_time_embedding",
"name": "ResNet block with time embedding",
"group": "Image and video",
"op": "SD ResNet block with time embedding",
"direction": "forward",
"model": "runwayml/stable-diffusion-v1-5",
"workloads": 16
},
{
"board": "L2",
"id": "067_patch_embed_to_joint_attention_input",
"name": "Patch embed to joint attention input",
"group": "Image and video",
"op": "SD3.5 patch embed + time / text conditioning",
"direction": "forward",
"model": "stabilityai/stable-diffusion-3.5-large",
"workloads": 16
},
{
"board": "L2",
"id": "068_gelu_approximate_feedforward_backward",
"name": "GELU approximate feedforward backward",
"group": "Backward pass",
"op": "GELU feed-forward backward",
"direction": "backward",
"model": "stabilityai/stable-diffusion-3.5-medium",
"workloads": 16
},
{
"board": "L2",
"id": "069_joint_transformer_block_residual_path",
"name": "Joint transformer block residual path",
"group": "Image and video",
"op": "SD3.5 joint transformer block residual path",
"direction": "forward",
"model": "stabilityai/stable-diffusion-3.5-medium",
"workloads": 16
},
{
"board": "L2",
"id": "070_basic_transformer_block",
"name": "Basic transformer block",
"group": "Image and video",
"op": "SDXL basic transformer block",
"direction": "forward",
"model": "stabilityai/stable-diffusion-xl-refiner-1.0",
"workloads": 16
},
{
"board": "L2",
"id": "071_edit_consistency_loss_with_perceptual_weighting",
"name": "Edit consistency loss with perceptual weighting",
"group": "Image and video",
"op": "edit-consistency perceptual loss",
"direction": "forward",
"model": "stepfun-ai/Step1X-Edit-v1p2",
"workloads": 16
},
{
"board": "L2",
"id": "072_region_aware_self_attention_with_edit_bias_backward",
"name": "Region aware self attention with edit bias backward",
"group": "Backward pass",
"op": "region-aware self-attention backward",
"direction": "backward",
"model": "stepfun-ai/Step1X-Edit-v1p2",
"workloads": 16
},
{
"board": "L2",
"id": "073_feedforward_mlp_backward",
"name": "Feedforward MLP backward",
"group": "Backward pass",
"op": "feed-forward MLP backward",
"direction": "backward",
"model": "stepfun-ai/Step1X-Edit",
"workloads": 16
},
{
"board": "L2",
"id": "074_sam_hq_mask_decoder_iou_hypernetwork_fusion_backward",
"name": "SAM hq mask decoder IoU hypernetwork fusion backward",
"group": "Backward pass",
"op": "SAM-HQ hypernetwork backward",
"direction": "backward",
"model": "syscv-community/sam-hq-vit-base",
"workloads": 16
},
{
"board": "L2",
"id": "075_sam_hq_mask_decoder_two_way_transformer",
"name": "SAM hq mask decoder two way transformer",
"group": "Image and video",
"op": "SAM-HQ two-way transformer block",
"direction": "forward",
"model": "syscv-community/sam-hq-vit-base",
"workloads": 16
},
{
"board": "L2",
"id": "076_sam_hq_vision_attention_with_relative_position_backward",
"name": "SAM hq vision attention with relative position backward",
"group": "Backward pass",
"op": "SAM-HQ relative-position attention backward",
"direction": "backward",
"model": "syscv-community/sam-hq-vit-base",
"workloads": 16
},
{
"board": "L2",
"id": "077_sam_hq_vision_encoder_window_partition_attention",
"name": "SAM hq vision encoder window partition attention",
"group": "Image and video",
"op": "SAM-HQ windowed attention block",
"direction": "forward",
"model": "syscv-community/sam-hq-vit-base",
"workloads": 16
},
{
"board": "L2",
"id": "078_fused_final_layer_upsample_with_adaptive_norm",
"name": "Fused final layer upsample with adaptive norm",
"group": "Image and video",
"op": "HunyuanImage final layer upsample + adaptive norm",
"direction": "forward",
"model": "tencent/HunyuanImage-3.0",
"workloads": 16
},
{
"board": "L2",
"id": "079_git_vision_encoder_layer",
"name": "Git vision encoder layer",
"group": "Image and video",
"op": "ViT encoder layer (MaskGIT)",
"direction": "forward",
"model": "valhalla/maskgit-imagenet-soft-code-34k",
"workloads": 16
},
{
"board": "L2",
"id": "080_moe_complete_layer_with_shared_expert_backward",
"name": "MoE complete layer with shared expert backward",
"group": "Backward pass",
"op": "MoE layer with shared expert backward",
"direction": "backward",
"model": "zai-org/GLM-4.5-Air",
"workloads": 16
},
{
"board": "L2",
"id": "081_moe_sparse_expert_dispatch",
"name": "MoE sparse expert dispatch",
"group": "MLP and MoE",
"op": "MoE block: sigmoid router + top-8 + experts + shared expert",
"direction": "forward",
"model": "zai-org/GLM-4.5-Air",
"workloads": 16
},
{
"board": "L2",
"id": "082_moe_layer_complete_forward_with_residual",
"name": "MoE layer complete forward with residual",
"group": "MLP and MoE",
"op": "full MoE layer: sigmoid + bias router, group top-8, 160 experts, shared expert, residual",
"direction": "forward",
"model": "zai-org/GLM-4.7",
"workloads": 16
},
{
"board": "Quant",
"id": "001_fp8_attention_output_projection",
"name": "FP8 attention output projection",
"group": "Projections and GEMM",
"op": "FP8 o_proj GEMM + bias (60x128 -> 7680)",
"direction": "forward",
"model": "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
"workloads": 16
},
{
"board": "Quant",
"id": "002_fp8_attention_qkv_projection",
"name": "FP8 attention QKV projection",
"group": "Projections and GEMM",
"op": "FP8 QKV projection (bias) + QK-RMSNorm",
"direction": "forward",
"model": "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
"workloads": 16
},
{
"board": "Quant",
"id": "003_fp8_mlp_gate_up_projection",
"name": "FP8 MLP gate up projection",
"group": "MLP and MoE",
"op": "FP8 gate+up GEMMs + SiLU gating (pre-quantized inputs)",
"direction": "forward",
"model": "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
"workloads": 16
},
{
"board": "Quant",
"id": "004_fp8_moe_expert_linear",
"name": "FP8 MoE expert linear",
"group": "MLP and MoE",
"op": "FP8 single-expert MLP: gate_up + SiLU + down, x routing weight",
"direction": "forward",
"model": "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
"workloads": 16
},
{
"board": "Quant",
"id": "005_fp8_moe_router_projection",
"name": "FP8 MoE router projection",
"group": "MLP and MoE",
"op": "FP8 MoE router GEMM (experts padded to 128)",
"direction": "forward",
"model": "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
"workloads": 16
},
{
"board": "Quant",
"id": "006_fp8_vision_attention_output_projection",
"name": "FP8 vision attention output projection",
"group": "Image and video",
"op": "FP8 vision o_proj GEMM (scales given)",
"direction": "forward",
"model": "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
"workloads": 16
},
{
"board": "Quant",
"id": "007_fp8_vision_attention_qkv_projection",
"name": "FP8 vision attention QKV projection",
"group": "Image and video",
"op": "FP8 vision QKV projection + bias",
"direction": "forward",
"model": "Qwen/Qwen3-VL-30B-A3B-Instruct-FP8",
"workloads": 16
},
{
"board": "Quant",
"id": "008_nvfp4_multimodal_embedding_projection",
"name": "NVFP4 multimodal embedding projection",
"group": "Image and video",
"op": "NVFP4 vision projection + merge into text embeddings",
"direction": "forward",
"model": "cybermotaz/Qwen3-VL-32B-Instruct-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "009_nvfp4_vision_temporal_patch_merge_with_projection",
"name": "NVFP4 vision temporal patch merge with projection",
"group": "Image and video",
"op": "NVFP4 temporal patch merge + projection",
"direction": "forward",
"model": "cybermotaz/Qwen3-VL-32B-Instruct-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "010_fp8_attention_output_projection",
"name": "FP8 attention output projection",
"group": "Projections and GEMM",
"op": "FP8 o_proj GEMM (16384 -> 7168)",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-R1-0528",
"workloads": 16
},
{
"board": "Quant",
"id": "011_fp8_moe_gate_routing",
"name": "FP8 MoE gate routing",
"group": "MLP and MoE",
"op": "FP8 router GEMM + sigmoid + bias + group top-k (8 groups / top-4 / top-8)",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-R1-0528",
"workloads": 16
},
{
"board": "Quant",
"id": "012_fp8_shared_expert_mlp",
"name": "FP8 shared expert MLP",
"group": "MLP and MoE",
"op": "FP8 shared-expert SwiGLU MLP",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-R1-0528",
"workloads": 16
},
{
"board": "Quant",
"id": "013_fp8_mla_kv_compression_projection",
"name": "FP8 MLA KV compression projection",
"group": "Projections and GEMM",
"op": "FP8 MLA KV path: kv_a GEMM -> RMSNorm -> kv_b GEMM",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-V3.1-Terminus",
"workloads": 16
},
{
"board": "Quant",
"id": "014_fp8_yarn_rope_embedding",
"name": "FP8 YaRN RoPE embedding",
"group": "Attention",
"op": "YaRN RoPE cos/sin table (DeepSeek)",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-V3.1-Terminus",
"workloads": 16
},
{
"board": "Quant",
"id": "015_fp8_mla_attention_output_projection",
"name": "FP8 MLA attention output projection",
"group": "Projections and GEMM",
"op": "FP8 MLA o_proj GEMM (pre-quantized act + weight)",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-V3",
"workloads": 16
},
{
"board": "Quant",
"id": "016_fp8_multi_latent_attention_qkv_projection",
"name": "FP8 multi latent attention QKV projection",
"group": "Projections and GEMM",
"op": "FP8 MLA Q and KV projections with both latent RMSNorms",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-V3",
"workloads": 16
},
{
"board": "Quant",
"id": "017_fp8_shared_expert_mlp",
"name": "FP8 shared expert MLP",
"group": "MLP and MoE",
"op": "FP8 shared-expert SwiGLU MLP (weight scales given)",
"direction": "forward",
"model": "deepseek-ai/DeepSeek-V3",
"workloads": 16
},
{
"board": "Quant",
"id": "018_nvfp4_attention_output_projection_with_residual",
"name": "NVFP4 attention output projection with residual",
"group": "Projections and GEMM",
"op": "NVFP4 o_proj GEMM + residual",
"direction": "forward",
"model": "mratsim/Nova-70B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "019_nvfp4_grouped_query_attention",
"name": "NVFP4 grouped query attention",
"group": "Attention",
"op": "NVFP4 GQA attention block: QKV, RoPE, attention, o_proj",
"direction": "forward",
"model": "mratsim/Nova-70B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "020_nvfp4_linear_layer",
"name": "NVFP4 linear layer",
"group": "Projections and GEMM",
"op": "NVFP4 linear (8192 x 8192)",
"direction": "forward",
"model": "mratsim/Nova-70B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "021_fp8_mamba2_in_projection",
"name": "FP8 Mamba2 in projection",
"group": "State space and conv",
"op": "FP8 Mamba2 in-projection GEMM",
"direction": "forward",
"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
"workloads": 16
},
{
"board": "Quant",
"id": "022_fp8_mamba2_out_projection",
"name": "FP8 Mamba2 out projection",
"group": "State space and conv",
"op": "FP8 Mamba2 out-projection GEMM (pre-quantized)",
"direction": "forward",
"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
"workloads": 16
},
{
"board": "Quant",
"id": "023_fp8_mamba2_ssm_discretization",
"name": "FP8 Mamba2 SSM discretization",
"group": "State space and conv",
"op": "FP8 Mamba2 SSM discretisation",
"direction": "forward",
"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8",
"workloads": 6
},
{
"board": "Quant",
"id": "024_nvfp4_attention_output_projection",
"name": "NVFP4 attention output projection",
"group": "Projections and GEMM",
"op": "NVFP4 o_proj GEMM (FP32 input)",
"direction": "forward",
"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "025_nvfp4_attention_qkv_projection",
"name": "NVFP4 attention QKV projection",
"group": "Projections and GEMM",
"op": "NVFP4 Q, K, V projections (FP32 input)",
"direction": "forward",
"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "026_nvfp4_mamba2_out_projection",
"name": "NVFP4 Mamba2 out projection",
"group": "State space and conv",
"op": "NVFP4 Mamba2 out-projection (FP4 act + weight)",
"direction": "forward",
"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "027_nvfp4_moe_router_projection",
"name": "NVFP4 MoE router projection",
"group": "MLP and MoE",
"op": "NVFP4 MoE router GEMM + sigmoid + bias + top-6",
"direction": "forward",
"model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "028_nvfp4_attention_qkv_projection_with_rope",
"name": "NVFP4 attention QKV projection with RoPE",
"group": "Projections and GEMM",
"op": "NVFP4 QKV projection + QK-RMSNorm + RoPE",
"direction": "forward",
"model": "nvidia/Qwen3-8B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "029_nvfp4_fused_mlp_silu_gating",
"name": "NVFP4 fused MLP SiLU gating",
"group": "MLP and MoE",
"op": "NVFP4 SwiGLU MLP (gate, up, down)",
"direction": "forward",
"model": "nvidia/Qwen3-8B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "030_nvfp4_fused_decoder_layer_pre_norm_attention",
"name": "NVFP4 fused decoder layer pre norm attention",
"group": "Attention",
"op": "NVFP4 decoder pre-norm + full attention block",
"direction": "forward",
"model": "nvidia/Qwen3-14B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "031_nvfp4_grouped_query_attention_with_kv_repeat",
"name": "NVFP4 grouped query attention with KV repeat",
"group": "Attention",
"op": "NVFP4-quantized attention core (FP4 QK^T and PV)",
"direction": "forward",
"model": "nvidia/Qwen3-14B-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "032_nvfp4_moe_expert_linear_with_gating",
"name": "NVFP4 MoE expert linear with gating",
"group": "MLP and MoE",
"op": "NVFP4 SwiGLU MLP (one expert-sized MLP, 2048 -> 5120)",
"direction": "forward",
"model": "nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4",
"workloads": 16
},
{
"board": "Quant",
"id": "033_nvfp4_moe_routing_with_topk_selection",
"name": "NVFP4 MoE routing with topk selection",
"group": "MLP and MoE",
"op": "NVFP4 router GEMM + softmax + top-10",
"direction": "forward",
"model": "nvidia/Qwen3-Next-80B-A3B-Thinking-NVFP4",
"workloads": 16
},
{
"board": "FlashInfer",
"id": "001_fused_add_rmsnorm_h2048",
"name": "Fused add RMSNorm h2048",
"group": "Normalization",
"op": "fused add + RMSNorm",
"direction": "forward",
"model": "flashinfer-trace tag: qwen3-30b-a3b",
"workloads": 7
},
{
"board": "FlashInfer",
"id": "002_fused_add_rmsnorm_h4096",
"name": "Fused add RMSNorm h4096",
"group": "Normalization",
"op": "fused add + RMSNorm",
"direction": "forward",
"model": "flashinfer-trace tag: llama-3.1-8b",
"workloads": 14
},
{
"board": "FlashInfer",
"id": "003_fused_add_rmsnorm_h7168",
"name": "Fused add RMSNorm h7168",
"group": "Normalization",
"op": "fused add + RMSNorm",
"direction": "forward",
"model": "flashinfer-trace tag: deepseek-v3, deepseek-r1",
"workloads": 8
},
{
"board": "FlashInfer",
"id": "004_gemm_n128_k2048",
"name": "GEMM n128 k2048",
"group": "Projections and GEMM",
"op": "dense GEMM C = A @ B^T (fp16)",
"direction": "forward",
"model": "flashinfer-trace tag: qwen3-30b-a3b (Qwen 3 30B A3B moe.gate)",
"workloads": 25
},
{
"board": "FlashInfer",
"id": "005_gemm_n256_k7168",
"name": "GEMM n256 k7168",
"group": "Projections and GEMM",
"op": "dense GEMM C = A @ B^T (fp16)",
"direction": "forward",
"model": "flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3 moe.gate)",
"workloads": 17
},
{
"board": "FlashInfer",
"id": "006_gemm_n2048_k4096",
"name": "GEMM n2048 k4096",
"group": "Projections and GEMM",
"op": "dense GEMM C = A @ B^T (fp16)",
"direction": "forward",
"model": "flashinfer-trace tag: qwen3-30b-a3b (Qwen 3 30B A3B attn.o_proj)",
"workloads": 29
},
{
"board": "FlashInfer",
"id": "007_gemm_n4096_k4096",
"name": "GEMM n4096 k4096",
"group": "Projections and GEMM",
"op": "dense GEMM C = A @ B^T (fp16)",
"direction": "forward",
"model": "flashinfer-trace tag: llama-3.1-8b (Llama 3.1 8B attn.o_proj)",
"workloads": 43
},
{
"board": "FlashInfer",
"id": "008_gemm_n4096_k14336",
"name": "GEMM n4096 k14336",
"group": "Projections and GEMM",
"op": "dense GEMM C = A @ B^T (fp16)",
"direction": "forward",
"model": "flashinfer-trace tag: llama-3.1-8b (Llama 3.1 8B mlp.down_proj)",
"workloads": 43
},
{
"board": "FlashInfer",
"id": "009_gemm_n5120_k2048",
"name": "GEMM n5120 k2048",
"group": "Projections and GEMM",
"op": "dense GEMM C = A @ B^T (fp16)",
"direction": "forward",
"model": "flashinfer-trace tag: qwen3-30b-a3b (Qwen 3 30B A3B attn.qkv_proj)",
"workloads": 25
},
{
"board": "FlashInfer",
"id": "010_gemm_n6144_k4096",
"name": "GEMM n6144 k4096",
"group": "Projections and GEMM",
"op": "dense GEMM C = A @ B^T (fp16)",
"direction": "forward",
"model": "flashinfer-trace tag: llama-3.1-8b (Llama 3.1 8B attn.qkv_proj)",
"workloads": 43
},
{
"board": "FlashInfer",
"id": "011_gemm_n28672_k4096",
"name": "GEMM n28672 k4096",
"group": "Projections and GEMM",
"op": "dense GEMM C = A @ B^T (fp16)",
"direction": "forward",
"model": "flashinfer-trace tag: llama-3.1-8b (Llama 3.1 8B mlp.gate_up_proj)",
"workloads": 43
},
{
"board": "FlashInfer",
"id": "012_gqa_paged_decode_h32_kv4_d128_ps1",
"name": "GQA paged decode h32 kv4 d128 ps1",
"group": "Attention",
"op": "GQA paged decode attention (page size 1)",
"direction": "forward",
"model": "flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B)",
"workloads": 48
},
{
"board": "FlashInfer",
"id": "013_gqa_paged_decode_h32_kv8_d128_ps1",
"name": "GQA paged decode h32 kv8 d128 ps1",
"group": "Attention",
"op": "GQA paged decode attention (page size 1)",
"direction": "forward",
"model": "flashinfer-trace tag: llama-3.1-8b (Llama-3.1-8B)",
"workloads": 48
},
{
"board": "FlashInfer",
"id": "014_gqa_paged_prefill_causal_h32_kv4_d128_ps1",
"name": "GQA paged prefill causal h32 kv4 d128 ps1",
"group": "Attention",
"op": "GQA paged prefill attention, causal (page size 1)",
"direction": "forward",
"model": "flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B during incremental prefill)",
"workloads": 30
},
{
"board": "FlashInfer",
"id": "015_gqa_paged_prefill_causal_h32_kv8_d128_ps1",
"name": "GQA paged prefill causal h32 kv8 d128 ps1",
"group": "Attention",
"op": "GQA paged prefill attention, causal (page size 1)",
"direction": "forward",
"model": "flashinfer-trace tag: llama-3.1-8b (Llama-3.1-8B during incremental prefill)",
"workloads": 38
},
{
"board": "FlashInfer",
"id": "016_gqa_ragged_prefill_causal_h32_kv4_d128",
"name": "GQA ragged prefill causal h32 kv4 d128",
"group": "Attention",
"op": "GQA ragged prefill attention, causal",
"direction": "forward",
"model": "flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B during total prefill)",
"workloads": 15
},
{
"board": "FlashInfer",
"id": "017_gqa_ragged_prefill_causal_h32_kv8_d128",
"name": "GQA ragged prefill causal h32 kv8 d128",
"group": "Attention",
"op": "GQA ragged prefill attention, causal",
"direction": "forward",
"model": "flashinfer-trace tag: llama-3.1-8b (Llama-3.1-8B during total prefill)",
"workloads": 21
},
{
"board": "FlashInfer",
"id": "018_mla_paged_decode_h16_ckv512_kpe64_ps1",
"name": "MLA paged decode h16 ckv512 kpe64 ps1",
"group": "Attention",
"op": "MLA paged decode attention (absorbed, ckv 512 + kpe 64, page size 1)",
"direction": "forward",
"model": "flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3 with tensor parallel size 8)",
"workloads": 47
},
{
"board": "FlashInfer",
"id": "019_mla_paged_prefill_causal_h16_ckv512_kpe64_ps1",
"name": "MLA paged prefill causal h16 ckv512 kpe64 ps1",
"group": "Attention",
"op": "MLA paged prefill attention, causal (absorbed)",
"direction": "forward",
"model": "flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3 during incremental prefill with tensor parallel size 8)",
"workloads": 38
},
{
"board": "FlashInfer",
"id": "020_moe_fp8_block_scale_ds_routing_topk8_ng8_kg4_e32_h7168_i2048",
"name": "MoE FP8 block scale ds routing topk8 ng8 kg4 e32 h7168 i2048",
"group": "MLP and MoE",
"op": "FP8 block-scale MoE: DeepSeek routing + 2 grouped GEMMs (32 local experts)",
"direction": "forward",
"model": "flashinfer-trace tag: deepseek-v3, deepseek-r1",
"workloads": 19
},
{
"board": "FlashInfer",
"id": "021_rmsnorm_h128",
"name": "RMSNorm h128",
"group": "Normalization",
"op": "RMSNorm",
"direction": "forward",
"model": "flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B. Epsilon is fixed at 1e-6)",
"workloads": 14
},
{
"board": "FlashInfer",
"id": "022_rmsnorm_h512",
"name": "RMSNorm h512",
"group": "Normalization",
"op": "RMSNorm",
"direction": "forward",
"model": "flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3/R1. Epsilon is fixed at 1e-6)",
"workloads": 8
},
{
"board": "FlashInfer",
"id": "023_rmsnorm_h1536",
"name": "RMSNorm h1536",
"group": "Normalization",
"op": "RMSNorm",
"direction": "forward",
"model": "flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3/R1. Epsilon is fixed at 1e-6)",
"workloads": 8
},
{
"board": "FlashInfer",
"id": "024_rmsnorm_h2048",
"name": "RMSNorm h2048",
"group": "Normalization",
"op": "RMSNorm",
"direction": "forward",
"model": "flashinfer-trace tag: qwen3-30b-a3b (Qwen3-30B-A3B. Epsilon is fixed at 1e-6)",
"workloads": 7
},
{
"board": "FlashInfer",
"id": "025_rmsnorm_h4096",
"name": "RMSNorm h4096",
"group": "Normalization",
"op": "RMSNorm",
"direction": "forward",
"model": "flashinfer-trace tag: llama-3.1-8b (Llama-3.1-8B. Epsilon is fixed at 1e-5)",
"workloads": 14
},
{
"board": "FlashInfer",
"id": "026_rmsnorm_h7168",
"name": "RMSNorm h7168",
"group": "Normalization",
"op": "RMSNorm",
"direction": "forward",
"model": "flashinfer-trace tag: deepseek-v3, deepseek-r1 (DeepSeek-V3/R1. Epsilon is fixed at 1e-6)",
"workloads": 8
}
]
}
{
"problems": [
{
"board": "L1",
"id": "001_attention_softmax_dropout_value_matmul_backward",
"name": "Attention softmax dropout value matmul backward",
"group": "Backward pass",
"op": "attention softmax + dropout + PV backward (GQA 80/8/128)",
"direction": "backward",
"model": "ByteDance/Seed-Seed-OSS-36B-Instruct",
"workloads": 16
},
{
"board": "L1",
"id": "002_vae_conv3x3_groupnorm_silu_residual_fused",
"name": "VAE conv3x3 GroupNorm SiLU residual fused",
"group": "Image and video",
"op": "VAE conv3x3 + GroupNorm + SiLU residual block",
"direction": "forward",
"model": "Efficient-Large-Model/Sana-Sprint-1.6B-1024px",
"workloads": 20
},
{
"board": "L1",
"id": "003_lm_head_projection_with_logit_slicing",
"name": "LM head projection with logit slicing",
"group": "Projections and GEMM",
"op": "LM head GEMM on the last N positions (logit slicing)",
"direction": "forward",
"model": "LGAI-EXAONE/EXAONE-4.0-1.2B",
"workloads": 16
},
{
"board": "L1",
"id": "004_attention_output_projection_with_reshape_backward",
"name": "Attention output projection with reshape backward",
"group": "Backward pass",
"op": "attention o_proj + reshape backward",
"direction": "backward",
"model": "LiquidAI/LFM2-1.2B",
"workloads": 16
},
{
"board": "L1",
"id": "005_conv_gated_projection_with_causal_conv",
"name": "Conv gated projection with causal conv",
"group": "State space and conv",
"op": "LFM2 short-conv block: in-proj, gate, causal depthwise conv1d, out-proj",
"direction": "forward",
"model": "LiquidAI/LFM2-1.2B",
"workloads": 16
},
{
"board": "L1",
"id": "006_hyena_depthwise_conv1d_split_gate",
"name": "Hyena depthwise Conv1D split gate",
"group": "State space and conv",
"op": "Hyena depthwise conv1d + split + gate",
"direction": "forward",
"model": "LongSafari/hyenadna-small-32k-seqlen-hf",
"workloads": 16
},
{
"board": "L1",
"id": "007_hyena_fft_size_padding_rfft",
"name": "Hyena FFT size padding RFFT",
"group": "State space and conv",
"op": "Hyena FFT padding + rfft",
"direction": "forward",
"model": "LongSafari/hyenadna-small-32k-seqlen-hf",
"workloads": 16
},
{
"board": "L1",
"id": "008_expert_output_weighted_index_add_accumulation",
"name": "Expert output weighted index add accumulation",
"group": "MLP and MoE",
"op": "MoE combine: weighted index_add of expert outputs",
"direction": "forward",
"model": "MiniMaxAI/MiniMax-M2",
"workloads": 16
},
{
"board": "L1",
"id": "009_expert_token_scatter_with_weighted_forward_backward",
"name": "Expert token scatter with weighted forward backward",
"group": "Backward pass",
"op": "MoE token scatter, weighted, forward + backward",
"direction": "backward",
"model": "MiniMaxAI/MiniMax-M2",
"workloads": 16
},
{
"board": "L1",
"id": "010_attention_value_projection_with_transpose",
"name": "Attention value projection with transpose",
"group": "Projections and GEMM",
"op": "V projection (5120 -> 1024) + reshape / transpose",
"direction": "forward",
"model": "NousResearch/DeepHermes-3-Mistral-24B-Preview",