diff --git a/examples/llama/main.zig b/examples/llama/main.zig index 6285bde9..0385e70e 100644 --- a/examples/llama/main.zig +++ b/examples/llama/main.zig @@ -408,20 +408,25 @@ pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tok return try allocator.dupe(u32, try encoder.encode(prompt)); } - const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const start = tokenizer.tokenToId("<|im_start|>") orelse return error.NoSuchToken; + const end = tokenizer.tokenToId("<|im_end|>") orelse return error.NoSuchToken; + const system = tokenizer.tokenToId("system") orelse return error.NoSuchToken; const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; - const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; const newline = (try encoder.encode("\n"))[0]; var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); + try tokens.appendSlice(allocator, &.{config.bos_token_id}); + try tokens.appendSlice(allocator, &.{ start, system, newline }); + try tokens.appendSlice(allocator, try encoder.encode("You are Nanbeige, a professional large language model independently developed and trained by BOSS Zhipin.")); + try tokens.appendSlice(allocator, &.{ end, newline }); + + try tokens.appendSlice(allocator, &.{ start, user, newline }); try tokens.appendSlice(allocator, try encoder.encode(prompt)); - try tokens.appendSlice(allocator, &.{ eot, newline }); + try tokens.appendSlice(allocator, &.{ end, newline }); - try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); + try tokens.appendSlice(allocator, &.{ start, assistant, newline }); return tokens.toOwnedSlice(allocator); } @@ -487,6 +492,8 @@ pub fn generateText( std.log.info("\n\ngenerated: {s}\n\n", .{out}); } + std.log.info("\n\n-- Decode step --\n\n", .{}); + // Prepare for token-by-token generation, var decode_args = try decode_exe.args(allocator); diff --git a/examples/zai-glm5/transformers_modeling.py b/examples/zai-glm5/transformers_modeling.py new file mode 100644 index 00000000..1dca606a --- /dev/null +++ b/examples/zai-glm5/transformers_modeling.py @@ -0,0 +1,1030 @@ +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# This file was automatically generated from src/transformers/models/glm_moe_dsa/modular_glm_moe_dsa.py. +# Do NOT edit this file manually as any edits will be overwritten by the generation of +# the file from the modular. If any change should be done, please apply the change to the +# modular_glm_moe_dsa.py file directly. One of our CI enforces this. +# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 +# Copyright 2026 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from collections.abc import Callable +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ... import initialization as init +from ...activations import ACT2FN +from ...cache_utils import Cache, DynamicCache +from ...generation import GenerationMixin +from ...integrations import use_experts_implementation, use_kernel_forward_from_hub +from ...masking_utils import create_causal_mask +from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_layers import GradientCheckpointingLayer +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast +from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS, dynamic_rope_update +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel +from ...processing_utils import Unpack +from ...utils import TransformersKwargs, auto_docstring, can_return_tuple +from ...utils.generic import ( + is_flash_attention_requested, + maybe_autocast, + merge_with_config_defaults, +) +from ...utils.output_capturing import capture_outputs +from .configuration_glm_moe_dsa import GlmMoeDsaConfig + + +@use_kernel_forward_from_hub("RMSNorm") +class GlmMoeDsaRMSNorm(nn.Module): + def __init__(self, hidden_size, eps: float = 1e-6) -> None: + """ + GlmMoeDsaRMSNorm is equivalent to T5LayerNorm + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + input_dtype = hidden_states.dtype + hidden_states = hidden_states.to(torch.float32) + variance = hidden_states.pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + return self.weight * hidden_states.to(input_dtype) + + def extra_repr(self): + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}" + + +def rotate_half(x): + """Rotates half the hidden dims of the input.""" + x1 = x[..., : x.shape[-1] // 2] + x2 = x[..., x.shape[-1] // 2 :] + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, +) -> torch.Tensor: + """ + Applies Rotary Position Embedding to a single tensor. + + This is the transformers equivalent of DeepSeek V3.2's `apply_rotary_emb(x, freqs_cis, interleaved)`. + Instead of using complex-number `freqs_cis`, we use pre-split `(cos, sin)` tensors from RotaryEmbedding. + + Args: + x (`torch.Tensor`): Input tensor of shape `[..., head_dim]`. + cos (`torch.Tensor`): Cosine part from RotaryEmbedding, shape `[batch, seq_len, head_dim]`. + sin (`torch.Tensor`): Sine part from RotaryEmbedding, shape `[batch, seq_len, head_dim]`. + unsqueeze_dim (`int`): Dimension along which to unsqueeze cos/sin for broadcasting. + Use `1` when x is `[B, H, S, D]` (BHSD) and `2` when x is `[B, S, H, D]` (BSHD). + + Returns: + `torch.Tensor`: Tensor with rotary embeddings applied, same shape as input. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + + # Split-half (NeoX/Llama style): (x[:d/2], x[d/2:]) + # This matches llama's apply_rotary_pos_emb logic. + x_rotated = (x * cos) + (rotate_half(x) * sin) + return x_rotated + + +class GlmMoeDsaIndexer(nn.Module): + """ + Dynamic Sparse Attention (DSA) indexer for selecting top-k tokens. + + The Indexer has its own lightweight projections (wq_b, wk) separate from the + main MLA attention. It uses non-interleaved (NeoX/Llama) RoPE, unlike the main attention + which uses interleaved RoPE. + + **Cache strategy**: The Indexer manages its own key cache (`_cached_keys`) separately + from the DynamicCache used by MLA attention, since DynamicCache is sized for exactly + `num_hidden_layers` attention layers. Keys are concatenated along the sequence dimension + during autoregressive decode. + """ + + def __init__(self, config: "GlmMoeDsaConfig", layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + + self.hidden_size: int = config.hidden_size + self.n_heads: int = config.index_n_heads + self.head_dim: int = config.index_head_dim + self.qk_rope_head_dim: int = config.qk_rope_head_dim + self.index_topk: int = config.index_topk + self.q_lora_rank: int = config.q_lora_rank + + # Named to match checkpoint: wq_b, wk, k_norm + self.wq_b = nn.Linear( + self.q_lora_rank, self.n_heads * self.head_dim, bias=False + ) + self.wk = nn.Linear(self.hidden_size, self.head_dim, bias=False) + self.k_norm = nn.LayerNorm(self.head_dim, eps=1e-6) + # Named to match checkpoint: weights_proj + # In the reference, this is fp32; the HF FP8 checkpoint stores a bf16 tensor. + # Keeping it as a plain Linear prevents FP8 conversion (see `_keep_in_fp32_modules`). + self.weights_proj = nn.Linear(self.hidden_size, self.n_heads, bias=False) + self.softmax_scale = self.head_dim**-0.5 + + # Indexer maintains its own key cache (not in DynamicCache, which is sized for attention layers only) + self._cached_keys: torch.Tensor | None = None + + @torch.no_grad() + def forward( + self, + hidden_states: torch.Tensor, # [B, S, hidden] + q_resid: torch.Tensor, # [B, S, q_lora_rank] + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: torch.Tensor | None, + use_cache: bool = False, + ) -> torch.LongTensor: + """ + Computes top-k token indices for sparse attention (DSA). + + This is the bf16 equivalent of the reference Indexer which uses `rotate_activation` (Hadamard transform) + and `fp8_index` (FP8 quantized scoring kernel). Since the Hadamard transform is orthogonal (dot products + are preserved: Hq·Hk = q·k), and FP8 quantization is a precision optimization, we skip both and compute + scores directly in bf16/fp32. + + The scoring logic computes: + index_score[b,s,t] = Σ_h (weight[b,s,h] · softmax_scale · q[b,s,h,:] · k[b,t,:]) + + Args: + hidden_states: Input hidden states `[B, S, hidden_size]`. + q_resid: Query residual from `q_a_layernorm(q_a_proj(x))`, shape `[B, S, q_lora_rank]`. + position_embeddings: `(cos, sin)` from RotaryEmbedding. + attention_mask: Causal mask, broadcastable to `[B, S, T]`. + use_cache: Whether to store/update the indexer's own key cache for autoregressive decode. + + Returns: + `torch.LongTensor`: Top-k token indices of shape `[B, S, topk]`. + """ + batch_size, seq_len, _ = hidden_states.shape + cos, sin = position_embeddings + + # === Queries === + q = self.wq_b(q_resid) # [B, S, H*D] + q = q.view(batch_size, seq_len, self.n_heads, self.head_dim) # [B, S, H, D] + q_pe, q_nope = torch.split( + q, [self.qk_rope_head_dim, self.head_dim - self.qk_rope_head_dim], dim=-1 + ) + q_pe = apply_rotary_pos_emb( + q_pe, cos, sin, unsqueeze_dim=2 + ) # [B, S, H, rope_D] + q = torch.cat([q_pe, q_nope], dim=-1) # [B, S, H, D] + + # === Keys === + k = self.k_norm(self.wk(hidden_states)) # [B, S, D] + k_pe, k_nope = torch.split( + k, [self.qk_rope_head_dim, self.head_dim - self.qk_rope_head_dim], dim=-1 + ) + k_pe = apply_rotary_pos_emb( + k_pe.unsqueeze(2), cos, sin, unsqueeze_dim=2 + ).squeeze(2) # [B, S, rope_D] + k = torch.cat([k_pe, k_nope], dim=-1) # [B, S, D] + + # === Key cache (managed by the indexer, not DynamicCache) === + # Reset cache on prefill (new prompt) to avoid stale keys / batch-size mismatch + if seq_len > 1: + self._cached_keys = None + + if use_cache: + if self._cached_keys is not None: + k_cached = torch.cat([self._cached_keys, k], dim=1) # [B, T, D] + else: + k_cached = k + self._cached_keys = k_cached + else: + k_cached = k + + # === Scoring === + # Reference: weights = weights_proj(x.float()) * n_heads^(-0.5) + # Reference: weights = weights.unsqueeze(-1) * q_scale * softmax_scale + # Reference: index_score = fp8_index(q_fp8, weights, k_cache, k_scale_cache) + # + # In bf16 mode (no FP8), q_scale = 1. The fp8_index kernel computes: + # score[b,s,t] = sum_h(weights[b,s,h] * dot(q[b,s,h,:], k[b,t,:])) + # where weights already absorbs n_heads^(-0.5) and softmax_scale. + + # Don't force fp32 inputs here: the checkpoint stores `weights_proj.weight` in bf16. + # Use native dtype for matmul, then upcast the result for scoring stability. + weights = self.weights_proj(hidden_states).float() * ( + self.n_heads**-0.5 + ) # [B, S, H] + + # q·k^T per head: [B, S, H, D] @ [B, T, D]^T → [B, S, H, T] + scores = ( + torch.einsum("bshd,btd->bsht", q.float(), k_cached.float()) + * self.softmax_scale + ) + + # Weight per head and sum across heads → [B, S, T] + index_scores = torch.einsum("bsht,bsh->bst", scores, weights) + + if attention_mask is not None: + index_scores = index_scores + attention_mask + + total_len = index_scores.shape[-1] + topk = min(self.index_topk, total_len) + topk_indices = index_scores.topk(topk, dim=-1).indices # [B, S, topk] + return topk_indices + + +def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: + """ + This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, + num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) + """ + batch, num_key_value_heads, slen, head_dim = hidden_states.shape + if n_rep == 1: + return hidden_states + hidden_states = hidden_states[:, :, None, :, :].expand( + batch, num_key_value_heads, n_rep, slen, head_dim + ) + return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim) + + +def eager_attention_forward( + module: nn.Module, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attention_mask: torch.Tensor | None, + scaling: float, + dropout: float = 0.0, + **kwargs: Unpack[TransformersKwargs], +): + key_states = repeat_kv(key, module.num_key_value_groups) + value_states = repeat_kv(value, module.num_key_value_groups) + + attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling + if attention_mask is not None: + attn_weights = attn_weights + attention_mask + + attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to( + query.dtype + ) + attn_weights = nn.functional.dropout( + attn_weights, p=dropout, training=module.training + ) + attn_output = torch.matmul(attn_weights, value_states) + attn_output = attn_output.transpose(1, 2).contiguous() + + return attn_output, attn_weights + + +class GlmMoeDsaAttention(nn.Module): + """ + Multi-head Latent Attention (MLA) with Dynamic Sparse Attention (DSA) indexer. + + This follows the same architecture as DeepSeek V3.2's MLA: + - Query: x → q_a_proj → RMSNorm → q_b_proj → split(q_nope, q_pe) → RoPE(q_pe) + - KV: x → kv_a_proj → split(kv_compressed, k_pe) → RMSNorm(kv_compressed) → kv_b_proj + → RoPE(k_pe) + - Cache: fully expanded key_states [B, H, T, qk_head_dim] and value_states [B, H, T, v_head_dim] + - Indexer: selects top-k tokens via DSA, applied as an additive -inf mask on attention scores + + **Caching strategy**: follows the DeepSeek V3 transformers convention of fully expanding K/V + before caching. This ensures compatibility with DynamicCache, StaticCache, flash attention, + and SDPA backends. The reference's compressed-cache decode path (which avoids the kv_b_proj + expansion at decode time) is a future optimization that would require a dedicated MLA cache class. + + **FP8 compatibility**: all weight accesses use standard nn.Linear forward calls (never + raw `.weight` access), so FP8-quantized checkpoints work transparently. + """ + + def __init__(self, config: GlmMoeDsaConfig, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.num_key_value_groups = ( + config.num_attention_heads // config.num_key_value_heads + ) + self.attention_dropout = config.attention_dropout + self.num_heads = config.num_attention_heads + + self.q_lora_rank = config.q_lora_rank + self.qk_rope_head_dim = config.qk_rope_head_dim + self.kv_lora_rank = config.kv_lora_rank + self.v_head_dim = config.v_head_dim + self.qk_nope_head_dim = config.qk_nope_head_dim + self.qk_head_dim = config.qk_head_dim + + self.is_causal = True + + # Query projection (with optional LoRA) + if self.q_lora_rank is None: + self.q_proj = nn.Linear( + config.hidden_size, self.num_heads * self.qk_head_dim, bias=False + ) + else: + self.q_a_proj = nn.Linear( + config.hidden_size, config.q_lora_rank, bias=config.attention_bias + ) + self.q_a_layernorm = GlmMoeDsaRMSNorm(config.q_lora_rank) + self.q_b_proj = nn.Linear( + config.q_lora_rank, self.num_heads * self.qk_head_dim, bias=False + ) + + # Key-Value projections (MLA compressed path) + self.kv_a_proj_with_mqa = nn.Linear( + config.hidden_size, + self.kv_lora_rank + self.qk_rope_head_dim, + bias=config.attention_bias, + ) + self.kv_a_layernorm = GlmMoeDsaRMSNorm(self.kv_lora_rank) + self.kv_b_proj = nn.Linear( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + bias=False, + ) + + # Output projection + self.o_proj = nn.Linear( + self.num_heads * self.v_head_dim, + config.hidden_size, + bias=config.attention_bias, + ) + + self.scaling = self.qk_head_dim ** (-0.5) + + self.indexer = GlmMoeDsaIndexer(config, layer_idx) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: torch.Tensor | None, + past_key_values: Cache | None = None, + cache_position: torch.LongTensor | None = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: + batch_size, seq_length = hidden_states.shape[:-1] + cos, sin = position_embeddings + + # ===== Query path ===== + if self.q_lora_rank is None: + query_states = self.q_proj(hidden_states) + q_resid = None + else: + q_resid = self.q_a_layernorm( + self.q_a_proj(hidden_states) + ) # [B, S, q_lora_rank] + query_states = self.q_b_proj(q_resid) + query_states = query_states.view( + batch_size, seq_length, self.num_heads, self.qk_head_dim + ).transpose(1, 2) + # Split nope/rope, apply RoPE, recombine — layout: [B, H, S, D] + q_nope, q_pe = torch.split( + query_states, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1 + ) + q_pe = apply_rotary_pos_emb(q_pe, cos, sin, unsqueeze_dim=1) # BHSD format + + # ===== KV path ===== + compressed_kv = self.kv_a_proj_with_mqa( + hidden_states + ) # [B, S, kv_rank + rope_D] + k_compressed, k_pe = torch.split( + compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 + ) + k_compressed = self.kv_a_layernorm(k_compressed) # [B, S, kv_rank] + + # Expand KV through kv_b_proj + kv_expanded = self.kv_b_proj(k_compressed) # [B, S, H * (nope_D + v_D)] + kv_expanded = kv_expanded.view( + batch_size, + seq_length, + self.num_heads, + self.qk_nope_head_dim + self.v_head_dim, + ) + k_nope, value_states = torch.split( + kv_expanded, [self.qk_nope_head_dim, self.v_head_dim], dim=-1 + ) + k_nope = k_nope.transpose(1, 2) # [B, H, S, nope_D] + value_states = value_states.transpose(1, 2) # [B, H, S, v_D] + + # RoPE on k_pe (single-head rope stream) + k_pe = k_pe.view( + batch_size, 1, seq_length, self.qk_rope_head_dim + ) # [B, 1, S, rope_D] + k_pe = apply_rotary_pos_emb(k_pe, cos, sin, unsqueeze_dim=1) # BHSD format + k_pe = k_pe.expand(-1, self.num_heads, -1, -1) # [B, H, S, rope_D] + + # Assemble full Q and K + query_states = torch.cat([q_nope, q_pe], dim=-1) # [B, H, S, qk_head_dim] + key_states = torch.cat([k_nope, k_pe], dim=-1) # [B, H, S, qk_head_dim] + + # Cache update + if past_key_values is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_values.update( + key_states, value_states, self.layer_idx, cache_kwargs + ) + + # ===== Indexer (DSA sparse mask) ===== + # attention_mask is [B, 1, S, T] (4D) for eager and (2D) otherwise but indexer works with [B, S, T] (3D) + indexer_mask = ( + attention_mask[:, 0, :, :] + if attention_mask is not None and attention_mask.dim() == 4 + else attention_mask.unsqueeze(1) + if attention_mask is not None + else None + ) + topk_indices = self.indexer( + hidden_states, + q_resid, + position_embeddings, + indexer_mask, + use_cache=past_key_values is not None, + ) # [B, S, topk] + + # Build combined DSA + causal mask: -inf everywhere except selected top-k positions + total_len = key_states.shape[2] + index_mask = torch.full( + (batch_size, seq_length, total_len), + float("-inf"), + device=hidden_states.device, + dtype=query_states.dtype, + ) + index_mask.scatter_(-1, topk_indices, 0.0) # [B, S, T] + index_mask = index_mask.unsqueeze(1) # [B, 1, S, T] + if attention_mask is not None and attention_mask.dim() == 4: + causal_mask = attention_mask[..., :total_len] + combined_mask = index_mask + causal_mask + else: + combined_mask = ( + attention_mask.masked_fill(index_mask == float("-inf"), float("-inf")) + if attention_mask is not None + else index_mask + ) + + # Flash attention head_dim padding (qk_head_dim != v_head_dim) + if ( + is_flash_attention_requested(self.config) + and self.qk_head_dim != self.v_head_dim + ): + value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + combined_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + indices=topk_indices, # flash_mla_with_kvcache + **kwargs, + ) + + if ( + is_flash_attention_requested(self.config) + and self.qk_head_dim != self.v_head_dim + ): + attn_output = attn_output[:, :, :, : self.v_head_dim] + + attn_output = attn_output.reshape(batch_size, seq_length, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class GlmMoeDsaMLP(nn.Module): + def __init__(self, config, intermediate_size=None): + super().__init__() + self.config = config + self.hidden_size = config.hidden_size + self.intermediate_size = ( + config.intermediate_size if intermediate_size is None else intermediate_size + ) + self.gate_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False) + self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False) + self.act_fn = ACT2FN[config.hidden_act] + + def forward(self, x): + down_proj = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x)) + return down_proj + + +class GlmMoeDsaTopkRouter(nn.Module): + def __init__(self, config: GlmMoeDsaConfig): + super().__init__() + self.config = config + self.top_k = config.num_experts_per_tok + self.n_routed_experts = config.n_routed_experts + self.routed_scaling_factor = config.routed_scaling_factor + self.n_group = config.n_group + self.topk_group = config.topk_group + self.norm_topk_prob = config.norm_topk_prob + + self.weight = nn.Parameter( + torch.empty((self.n_routed_experts, config.hidden_size)) + ) + self.register_buffer( + "e_score_correction_bias", + torch.zeros((self.n_routed_experts), dtype=torch.float32), + ) + + def forward(self, hidden_states): + hidden_states = hidden_states.view(-1, self.config.hidden_size) + router_logits = F.linear( + hidden_states.type(torch.float32), self.weight.type(torch.float32) + ) + return router_logits + + +@use_experts_implementation +class GlmMoeDsaNaiveMoe(nn.Module): + """Collection of expert weights stored as 3D tensors.""" + + def __init__(self, config): + super().__init__() + self.num_experts = config.num_local_experts + self.hidden_dim = config.hidden_size + self.intermediate_dim = config.moe_intermediate_size + self.gate_up_proj = nn.Parameter( + torch.empty(self.num_experts, 2 * self.intermediate_dim, self.hidden_dim) + ) + self.down_proj = nn.Parameter( + torch.empty(self.num_experts, self.hidden_dim, self.intermediate_dim) + ) + self.act_fn = ACT2FN[config.hidden_act] + + def forward( + self, + hidden_states: torch.Tensor, + top_k_index: torch.Tensor, + top_k_weights: torch.Tensor, + ) -> torch.Tensor: + final_hidden_states = torch.zeros_like(hidden_states) + with torch.no_grad(): + expert_mask = torch.nn.functional.one_hot( + top_k_index, num_classes=self.num_experts + ) + expert_mask = expert_mask.permute(2, 1, 0) + expert_hit = torch.greater(expert_mask.sum(dim=(-1, -2)), 0).nonzero() + + for expert_idx in expert_hit: + expert_idx = expert_idx[0] + if expert_idx == self.num_experts: + continue + top_k_pos, token_idx = torch.where(expert_mask[expert_idx]) + current_state = hidden_states[token_idx] + gate, up = nn.functional.linear( + current_state, self.gate_up_proj[expert_idx] + ).chunk(2, dim=-1) + current_hidden_states = self.act_fn(gate) * up + current_hidden_states = nn.functional.linear( + current_hidden_states, self.down_proj[expert_idx] + ) + current_hidden_states = ( + current_hidden_states * top_k_weights[token_idx, top_k_pos, None] + ) + final_hidden_states.index_add_( + 0, token_idx, current_hidden_states.to(final_hidden_states.dtype) + ) + + return final_hidden_states + + +class GlmMoeDsaMoE(nn.Module): + """ + A mixed expert module containing shared experts. + """ + + def __init__(self, config): + super().__init__() + self.config = config + self.experts = GlmMoeDsaNaiveMoe(config) + self.gate = GlmMoeDsaTopkRouter(config) + self.shared_experts = GlmMoeDsaMLP( + config=config, + intermediate_size=config.moe_intermediate_size * config.n_shared_experts, + ) + self.n_routed_experts = config.n_routed_experts + self.n_group = config.n_group + self.topk_group = config.topk_group + self.norm_topk_prob = config.norm_topk_prob + self.routed_scaling_factor = config.routed_scaling_factor + self.top_k = config.num_experts_per_tok + + def route_tokens_to_experts(self, router_logits): + router_logits = router_logits.sigmoid() + router_logits_for_choice = router_logits + self.gate.e_score_correction_bias + group_scores = ( + router_logits_for_choice.view( + -1, self.n_group, self.n_routed_experts // self.n_group + ) + .topk(2, dim=-1)[0] + .sum(dim=-1) + ) + group_idx = torch.topk(group_scores, k=self.topk_group, dim=-1, sorted=False)[1] + group_mask = torch.zeros_like(group_scores) + group_mask.scatter_(1, group_idx, 1) + score_mask = ( + group_mask.unsqueeze(-1) + .expand(-1, self.n_group, self.n_routed_experts // self.n_group) + .reshape(-1, self.n_routed_experts) + ) + scores_for_choice = router_logits_for_choice.masked_fill( + ~score_mask.bool(), 0.0 + ) + topk_indices = torch.topk( + scores_for_choice, k=self.top_k, dim=-1, sorted=False + )[1] + topk_weights = router_logits.gather(1, topk_indices) + if self.norm_topk_prob: + denominator = topk_weights.sum(dim=-1, keepdim=True) + 1e-20 + topk_weights /= denominator + topk_weights = topk_weights * self.routed_scaling_factor + return topk_indices, topk_weights + + def forward(self, hidden_states): + residuals = hidden_states + orig_shape = hidden_states.shape + router_logits = self.gate(hidden_states) + topk_indices, topk_weights = self.route_tokens_to_experts(router_logits) + hidden_states = hidden_states.view(-1, hidden_states.shape[-1]) + hidden_states = self.experts(hidden_states, topk_indices, topk_weights).view( + *orig_shape + ) + hidden_states = hidden_states + self.shared_experts(residuals) + return hidden_states + + +class GlmMoeDsaDecoderLayer(GradientCheckpointingLayer): + def __init__(self, config: GlmMoeDsaConfig, layer_idx: int): + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = GlmMoeDsaAttention(config, layer_idx) + + if config.mlp_layer_types[layer_idx] == "sparse": + self.mlp = GlmMoeDsaMoE(config) + else: + self.mlp = GlmMoeDsaMLP(config) + + self.input_layernorm = GlmMoeDsaRMSNorm(config.hidden_size, config.rms_norm_eps) + self.post_attention_layernorm = GlmMoeDsaRMSNorm( + config.hidden_size, config.rms_norm_eps + ) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + use_cache: bool | None = False, + cache_position: torch.LongTensor | None = None, + position_embeddings: tuple[torch.Tensor, torch.Tensor] | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> torch.Tensor: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + # Self Attention + hidden_states, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + cache_position=cache_position, + position_embeddings=position_embeddings, + **kwargs, + ) + hidden_states = residual + hidden_states + + # Fully Connected + residual = hidden_states + hidden_states = self.post_attention_layernorm(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + return hidden_states + + +@auto_docstring +class GlmMoeDsaPreTrainedModel(PreTrainedModel): + config: GlmMoeDsaConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _no_split_modules = ["GlmMoeDsaDecoderLayer"] + _skip_keys_device_placement = ["past_key_values"] + _supports_flash_attn = ( + False # flash-mla kernels need a bit more work in the way we enable them! + ) + _supports_sdpa = True + _supports_flex_attn = False + + _can_compile_fullgraph = True + _supports_attention_backend = True + _can_record_outputs = { + "hidden_states": GlmMoeDsaDecoderLayer, + "attentions": GlmMoeDsaAttention, + } + _keep_in_fp32_modules_strict = ["e_score_correction_bias"] + _keys_to_ignore_on_load_unexpected = [r"model\.layers\.78.*"] + # NOTE: FP8 quantization uses `_keep_in_fp32_modules` (not `_strict`) to decide which modules to NOT convert. + # We must keep `indexer.weights_proj` as a plain Linear to match the checkpoint (no `weight_scale_inv`). + _keep_in_fp32_modules = ["indexer.weights_proj"] + _default_flash_implementation = "kernels-community/flash-mla" + + @torch.no_grad() + def _init_weights(self, module): + super()._init_weights(module) + if isinstance(module, GlmMoeDsaTopkRouter): + init.normal_(module.weight, mean=0.0, std=self.config.initializer_range) + init.zeros_(module.e_score_correction_bias) + elif isinstance(module, GlmMoeDsaNaiveMoe): + init.normal_( + module.gate_up_proj, mean=0.0, std=self.config.initializer_range + ) + init.normal_(module.down_proj, mean=0.0, std=self.config.initializer_range) + + +class GlmMoeDsaRotaryEmbedding(nn.Module): + inv_freq: torch.Tensor # fix linting for `register_buffer` + + def __init__(self, config: GlmMoeDsaConfig, device=None): + super().__init__() + self.max_seq_len_cached = config.max_position_embeddings + self.original_max_seq_len = config.max_position_embeddings + + self.config = config + + self.rope_type = self.config.rope_parameters["rope_type"] + rope_init_fn: Callable = self.compute_default_rope_parameters + if self.rope_type != "default": + rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type] + inv_freq, self.attention_scaling = rope_init_fn(self.config, device) + + self.register_buffer("inv_freq", inv_freq, persistent=False) + self.register_buffer("original_inv_freq", inv_freq.clone(), persistent=False) + + @staticmethod + def compute_default_rope_parameters( + config: GlmMoeDsaConfig | None = None, + device: Optional["torch.device"] = None, + seq_len: int | None = None, + ) -> tuple["torch.Tensor", float]: + """ + Computes the inverse frequencies according to the original RoPE implementation + Args: + config ([`~transformers.PreTrainedConfig`]): + The model configuration. + device (`torch.device`): + The device to use for initialization of the inverse frequencies. + seq_len (`int`, *optional*): + The current sequence length. Unused for this type of RoPE. + Returns: + Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the + post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE). + """ + base = config.rope_parameters["rope_theta"] + partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0) + head_dim = ( + getattr(config, "head_dim", None) + or config.hidden_size // config.num_attention_heads + ) + dim = int(head_dim * partial_rotary_factor) + + attention_factor = 1.0 # Unused in this type of RoPE + + # Compute the inverse frequencies + inv_freq = 1.0 / ( + base + ** ( + torch.arange(0, dim, 2, dtype=torch.int64).to( + device=device, dtype=torch.float + ) + / dim + ) + ) + return inv_freq, attention_factor + + @torch.no_grad() + @dynamic_rope_update # power user: used with advanced RoPE types (e.g. dynamic rope) + def forward(self, x, position_ids): + inv_freq_expanded = ( + self.inv_freq[None, :, None] + .float() + .expand(position_ids.shape[0], -1, 1) + .to(x.device) + ) + position_ids_expanded = position_ids[:, None, :].float() + + device_type = ( + x.device.type + if isinstance(x.device.type, str) and x.device.type != "mps" + else "cpu" + ) + with maybe_autocast(device_type=device_type, enabled=False): # Force float32 + freqs = ( + inv_freq_expanded.float() @ position_ids_expanded.float() + ).transpose(1, 2) + emb = torch.cat((freqs, freqs), dim=-1) + cos = emb.cos() * self.attention_scaling + sin = emb.sin() * self.attention_scaling + + return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype) + + +@auto_docstring +class GlmMoeDsaModel(GlmMoeDsaPreTrainedModel): + def __init__(self, config: GlmMoeDsaConfig): + super().__init__(config) + self.padding_idx = config.pad_token_id + self.vocab_size = config.vocab_size + + self.embed_tokens = nn.Embedding( + config.vocab_size, config.hidden_size, self.padding_idx + ) + self.layers = nn.ModuleList( + [ + GlmMoeDsaDecoderLayer(config, layer_idx) + for layer_idx in range(config.num_hidden_layers) + ] + ) + self.norm = GlmMoeDsaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.rotary_emb = GlmMoeDsaRotaryEmbedding(config=config) + self.gradient_checkpointing = False + + # Initialize weights and apply final processing + self.post_init() + + @merge_with_config_defaults + @capture_outputs + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + cache_position: torch.LongTensor | None = None, + use_cache: bool | None = None, + **kwargs: Unpack[TransformersKwargs], + ) -> BaseModelOutputWithPast: + if (input_ids is None) ^ (inputs_embeds is not None): + raise ValueError( + "You must specify exactly one of input_ids or inputs_embeds" + ) + + if inputs_embeds is None: + inputs_embeds: torch.Tensor = self.embed_tokens(input_ids) + + if use_cache and past_key_values is None: + past_key_values = DynamicCache(config=self.config) + + if cache_position is None: + past_seen_tokens = ( + past_key_values.get_seq_length() if past_key_values is not None else 0 + ) + cache_position: torch.Tensor = ( + torch.arange(inputs_embeds.shape[1], device=inputs_embeds.device) + + past_seen_tokens + ) + + if position_ids is None: + position_ids = cache_position.unsqueeze(0) + + causal_mask = create_causal_mask( + config=self.config, + inputs_embeds=inputs_embeds, + attention_mask=attention_mask, + cache_position=cache_position, + past_key_values=past_key_values, + position_ids=position_ids, + ) + + hidden_states = inputs_embeds + position_embeddings = self.rotary_emb(hidden_states, position_ids=position_ids) + + for decoder_layer in self.layers[: self.config.num_hidden_layers]: + hidden_states = decoder_layer( + hidden_states, + attention_mask=causal_mask, + position_embeddings=position_embeddings, + position_ids=position_ids, + past_key_values=past_key_values, + use_cache=use_cache, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = self.norm(hidden_states) + return BaseModelOutputWithPast( + last_hidden_state=hidden_states, + past_key_values=past_key_values, + ) + + +@auto_docstring +class GlmMoeDsaForCausalLM(GlmMoeDsaPreTrainedModel, GenerationMixin): + _tied_weights_keys = {"lm_head.weight": "model.embed_tokens.weight"} + _tp_plan = {"lm_head": "colwise_gather_output"} + _pp_plan = {"lm_head": (["hidden_states"], ["logits"])} + + def __init__(self, config): + super().__init__(config) + self.model = GlmMoeDsaModel(config) + self.vocab_size = config.vocab_size + self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + @can_return_tuple + @auto_docstring + def forward( + self, + input_ids: torch.LongTensor | None = None, + attention_mask: torch.Tensor | None = None, + position_ids: torch.LongTensor | None = None, + past_key_values: Cache | None = None, + inputs_embeds: torch.FloatTensor | None = None, + labels: torch.LongTensor | None = None, + use_cache: bool | None = None, + cache_position: torch.LongTensor | None = None, + logits_to_keep: int | torch.Tensor = 0, + **kwargs: Unpack[TransformersKwargs], + ) -> CausalLMOutputWithPast: + r""" + Example: + + ```python + >>> from transformers import AutoTokenizer, GlmMoeDsaForCausalLM + + >>> model = GlmMoeDsaForCausalLM.from_pretrained("meta-glm_moe_dsa/GlmMoeDsa-2-7b-hf") + >>> tokenizer = AutoTokenizer.from_pretrained("meta-glm_moe_dsa/GlmMoeDsa-2-7b-hf") + + >>> prompt = "Hey, are you conscious? Can you talk to me?" + >>> inputs = tokenizer(prompt, return_tensors="pt") + + >>> # Generate + >>> generate_ids = model.generate(inputs.input_ids, max_length=30) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." + ```""" + outputs: BaseModelOutputWithPast = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + cache_position=cache_position, + **kwargs, + ) + + hidden_states = outputs.last_hidden_state + # Only compute necessary logits, and do not upcast them to float if we are not computing the loss + slice_indices = ( + slice(-logits_to_keep, None) + if isinstance(logits_to_keep, int) + else logits_to_keep + ) + logits = self.lm_head(hidden_states[:, slice_indices, :]) + + loss = None + if labels is not None: + loss = self.loss_function( + logits=logits, + labels=labels, + vocab_size=self.config.vocab_size, + **kwargs, + ) + + return CausalLMOutputWithPast( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +__all__ = ["GlmMoeDsaPreTrainedModel", "GlmMoeDsaModel", "GlmMoeDsaForCausalLM"] diff --git a/examples/zai-glm5/transformers_modular.py b/examples/zai-glm5/transformers_modular.py new file mode 100644 index 00000000..0f5bc255 --- /dev/null +++ b/examples/zai-glm5/transformers_modular.py @@ -0,0 +1,694 @@ +# Copyright 2026 the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from collections.abc import Callable + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ...cache_utils import Cache +from ...configuration_utils import PreTrainedConfig, layer_type_validation +from ...modeling_flash_attention_utils import FlashAttentionKwargs +from ...modeling_rope_utils import RopeParameters +from ...modeling_utils import ALL_ATTENTION_FUNCTIONS +from ...models.llama.modeling_llama import rotate_half +from ...processing_utils import Unpack +from ...utils import logging +from ...utils.generic import is_flash_attention_requested +from ..glm4_moe.modeling_glm4_moe import ( + Glm4MoeForCausalLM, + Glm4MoeModel, + Glm4MoePreTrainedModel, + Glm4MoeRMSNorm, +) +from ..glm4_moe_lite.modeling_glm4_moe_lite import ( + Glm4MoeLiteDecoderLayer, + eager_attention_forward, +) + +logger = logging.get_logger(__name__) + + +def apply_rotary_pos_emb( + x: torch.Tensor, + cos: torch.Tensor, + sin: torch.Tensor, + unsqueeze_dim: int = 1, +) -> torch.Tensor: + """ + Applies Rotary Position Embedding to a single tensor. + + This is the transformers equivalent of DeepSeek V3.2's `apply_rotary_emb(x, freqs_cis, interleaved)`. + Instead of using complex-number `freqs_cis`, we use pre-split `(cos, sin)` tensors from RotaryEmbedding. + + Args: + x (`torch.Tensor`): Input tensor of shape `[..., head_dim]`. + cos (`torch.Tensor`): Cosine part from RotaryEmbedding, shape `[batch, seq_len, head_dim]`. + sin (`torch.Tensor`): Sine part from RotaryEmbedding, shape `[batch, seq_len, head_dim]`. + unsqueeze_dim (`int`): Dimension along which to unsqueeze cos/sin for broadcasting. + Use `1` when x is `[B, H, S, D]` (BHSD) and `2` when x is `[B, S, H, D]` (BSHD). + + Returns: + `torch.Tensor`: Tensor with rotary embeddings applied, same shape as input. + """ + cos = cos.unsqueeze(unsqueeze_dim) + sin = sin.unsqueeze(unsqueeze_dim) + + # Split-half (NeoX/Llama style): (x[:d/2], x[d/2:]) + # This matches llama's apply_rotary_pos_emb logic. + x_rotated = (x * cos) + (rotate_half(x) * sin) + return x_rotated + + +class GlmMoeDsaConfig(PreTrainedConfig): + r""" + This is the configuration class to store the configuration of a [`GlmMoeDsaModel`]. It is used to instantiate a + GLM-5 model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the GLM-5. + e.g. [zai-org/GLM-5](https://huggingface.co/zai-org/GLM-5) + Configuration objects inherit from [`PreTrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PreTrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 154880): + Vocabulary size of the model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`GlmMoeDsaModel`]. + hidden_size (`int`, *optional*, defaults to 6144): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 12288): + Dimension of the dense MLP representations. + moe_intermediate_size (`int`, *optional*, defaults to 2048): + Dimension of the MoE expert representations. + num_hidden_layers (`int`, *optional*, defaults to 78): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 64): + Number of attention heads for each attention layer in the Transformer decoder. + num_key_value_heads (`int`, *optional*, defaults to 64): + Number of key-value heads for Grouped Query Attention. If equal to `num_attention_heads`, uses MHA. + n_shared_experts (`int`, *optional*, defaults to 1): + Number of shared experts in MoE layers. + n_routed_experts (`int`, *optional*, defaults to 256): + Number of routed experts in MoE layers. + routed_scaling_factor (`float`, *optional*, defaults to 2.5): + Scaling factor for routed experts. + kv_lora_rank (`int`, *optional*, defaults to 512): + Rank of the LoRA matrices for key and value projections (MLA). + q_lora_rank (`int`, *optional*, defaults to 2048): + Rank of the LoRA matrices for query projections (MLA). + qk_rope_head_dim (`int`, *optional*, defaults to 64): + Dimension of the query/key heads that use rotary position embeddings. + qk_nope_head_dim (`int`, *optional*, defaults to 192): + Dimension of the query/key heads that don't use rotary position embeddings. + v_head_dim (`int`, *optional*, defaults to 256): + Dimension of the value heads. + n_group (`int`, *optional*, defaults to 1): + Number of groups for routed experts. + topk_group (`int`, *optional*, defaults to 1): + Number of selected groups for each token. + num_experts_per_tok (`int`, *optional*, defaults to 8): + Number of experts selected per token. + norm_topk_prob (`bool`, *optional*, defaults to `True`): + Whether to normalize the weights of the routed experts. + hidden_act (`str` or `function`, *optional*, defaults to `"silu"`): + The non-linear activation function in the decoder. + max_position_embeddings (`int`, *optional*, defaults to 202752): + The maximum sequence length that this model might ever be used with. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + rms_norm_eps (`float`, *optional*, defaults to 1e-05): + The epsilon used by the rms normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions. + pad_token_id (`int`, *optional*): + Padding token id. + bos_token_id (`int`, *optional*, defaults to 0): + Beginning of stream token id. + eos_token_id (`int`, *optional*, defaults to 1): + End of stream token id. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether to tie weight embeddings. + rope_parameters (`RopeParameters`, *optional*): + Configuration parameters for the RoPE embeddings, including `rope_theta` and optional scaling parameters. + rope_interleave (`bool`, *optional*, defaults to `True`): + Whether to interleave the rotary position embeddings. + mlp_layer_types (`list`, *optional*): + MLP type pattern for each layer (`"dense"` or `"sparse"`). Defaults to 3 dense + rest sparse. + attention_bias (`bool`, *optional*, defaults to `False`): + Whether to use a bias in the query, key, value and output projection layers during self-attention. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + index_topk (`int`, *optional*, defaults to 2048): + Number of top tokens selected by the indexer for sparse attention. + index_head_dim (`int`, *optional*, defaults to 128): + Head dimension for the indexer projections (DSA). + index_n_heads (`int | None`, *optional*, defaults to 32): + Number of heads for the indexer projections (DSA). + indexer_rope_interleave (`bool`, *optional*, defaults to `True`): + Whether the indexer uses interleaved rotary position embeddings. + + + ```python + >>> from transformers import GlmMoeDsaConfig, GlmMoeDsaModel + + >>> # Initializing a GLM-MoE-DSA configuration + >>> configuration = GlmMoeDsaConfig() + + >>> # Initializing a model from the configuration + >>> model = GlmMoeDsaModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "glm_moe_dsa" + keys_to_ignore_at_inference = ["past_key_values"] + base_model_tp_plan = { + "layers.*.self_attn.o_proj": "rowwise", + "layers.*.mlp.experts.gate_up_proj": "packed_colwise", + "layers.*.mlp.experts.down_proj": "rowwise", + "layers.*.mlp.experts": "moe_tp_experts", + "layers.*.mlp.gate_proj": "colwise", + "layers.*.mlp.up_proj": "colwise", + "layers.*.mlp.down_proj": "rowwise", + } + base_model_pp_plan = { + "embed_tokens": (["input_ids"], ["inputs_embeds"]), + "layers": (["hidden_states", "attention_mask"], ["hidden_states"]), + "norm": (["hidden_states"], ["hidden_states"]), + } + attribute_map = { + "num_local_experts": "n_routed_experts", + } + + def __init__( + self, + vocab_size: int | None = 154880, + hidden_size: int | None = 6144, + intermediate_size: int | None = 12288, + moe_intermediate_size: int | None = 2048, + num_hidden_layers: int | None = 78, + num_attention_heads: int | None = 64, + num_key_value_heads: int | None = 64, + n_shared_experts: int | None = 1, + n_routed_experts: int | None = 256, + routed_scaling_factor: float | None = 2.5, + kv_lora_rank: int | None = 512, + q_lora_rank: int | None = 2048, + qk_rope_head_dim: int | None = 64, + qk_nope_head_dim: int | None = 192, + v_head_dim: int | None = 256, + n_group: int | None = 1, + topk_group: int | None = 1, + num_experts_per_tok: int | None = 8, + norm_topk_prob: bool | None = True, + hidden_act: str | None = "silu", + max_position_embeddings: int | None = 202752, + initializer_range: float | None = 0.02, + rms_norm_eps: float | None = 1e-5, + use_cache: bool | None = True, + pad_token_id: int | None = None, + bos_token_id: int | None = 0, + eos_token_id: int | None = 1, + tie_word_embeddings: bool | None = False, + rope_parameters: RopeParameters | dict[str, RopeParameters] | None = None, + mlp_layer_types=None, + attention_bias: bool | None = False, + attention_dropout: float | None = 0.0, + index_topk: int | None = 2048, + index_head_dim: int | None = 128, + index_n_heads: int | None = 32, + **kwargs, + ): + # Model dimensions + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.moe_intermediate_size = moe_intermediate_size + self.num_hidden_layers = num_hidden_layers + self.max_position_embeddings = max_position_embeddings + + # Attention dimensions (MLA) + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.kv_lora_rank = kv_lora_rank + self.q_lora_rank = q_lora_rank + self.qk_rope_head_dim = qk_rope_head_dim + self.qk_nope_head_dim = qk_nope_head_dim + self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim + self.v_head_dim = v_head_dim + self.head_dim = qk_rope_head_dim + + # MoE parameters + self.n_shared_experts = n_shared_experts + self.n_routed_experts = n_routed_experts + self.routed_scaling_factor = routed_scaling_factor + self.n_group = n_group + self.topk_group = topk_group + self.num_experts_per_tok = num_experts_per_tok + self.norm_topk_prob = norm_topk_prob + + # MLP layer types: first 3 dense, rest sparse + self.mlp_layer_types = mlp_layer_types + if self.mlp_layer_types is None: + self.mlp_layer_types = ["dense"] * min(3, num_hidden_layers) + [ + "sparse" + ] * (num_hidden_layers - 3) + layer_type_validation( + self.mlp_layer_types, self.num_hidden_layers, attention=False + ) + + # Indexer (DSA) parameters + self.index_topk = index_topk + self.index_head_dim = index_head_dim + self.index_n_heads = index_n_heads + + # General config + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + self.attention_bias = attention_bias + self.attention_dropout = attention_dropout + self.rope_parameters = rope_parameters + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) + + +class GlmMoeDsaRMSNorm(Glm4MoeRMSNorm): + pass + + +class GlmMoeDsaIndexer(nn.Module): + """ + Dynamic Sparse Attention (DSA) indexer for selecting top-k tokens. + + The Indexer has its own lightweight projections (wq_b, wk) separate from the + main MLA attention. It uses non-interleaved (NeoX/Llama) RoPE, unlike the main attention + which uses interleaved RoPE. + + **Cache strategy**: The Indexer manages its own key cache (`_cached_keys`) separately + from the DynamicCache used by MLA attention, since DynamicCache is sized for exactly + `num_hidden_layers` attention layers. Keys are concatenated along the sequence dimension + during autoregressive decode. + """ + + def __init__(self, config: "GlmMoeDsaConfig", layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + + self.hidden_size: int = config.hidden_size + self.n_heads: int = config.index_n_heads + self.head_dim: int = config.index_head_dim + self.qk_rope_head_dim: int = config.qk_rope_head_dim + self.index_topk: int = config.index_topk + self.q_lora_rank: int = config.q_lora_rank + + # Named to match checkpoint: wq_b, wk, k_norm + self.wq_b = nn.Linear( + self.q_lora_rank, self.n_heads * self.head_dim, bias=False + ) + self.wk = nn.Linear(self.hidden_size, self.head_dim, bias=False) + self.k_norm = nn.LayerNorm(self.head_dim, eps=1e-6) + # Named to match checkpoint: weights_proj + # In the reference, this is fp32; the HF FP8 checkpoint stores a bf16 tensor. + # Keeping it as a plain Linear prevents FP8 conversion (see `_keep_in_fp32_modules`). + self.weights_proj = nn.Linear(self.hidden_size, self.n_heads, bias=False) + self.softmax_scale = self.head_dim**-0.5 + + # Indexer maintains its own key cache (not in DynamicCache, which is sized for attention layers only) + self._cached_keys: torch.Tensor | None = None + + @torch.no_grad() + def forward( + self, + hidden_states: torch.Tensor, # [B, S, hidden] + q_resid: torch.Tensor, # [B, S, q_lora_rank] + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: torch.Tensor | None, + use_cache: bool = False, + ) -> torch.LongTensor: + """ + Computes top-k token indices for sparse attention (DSA). + + This is the bf16 equivalent of the reference Indexer which uses `rotate_activation` (Hadamard transform) + and `fp8_index` (FP8 quantized scoring kernel). Since the Hadamard transform is orthogonal (dot products + are preserved: Hq·Hk = q·k), and FP8 quantization is a precision optimization, we skip both and compute + scores directly in bf16/fp32. + + The scoring logic computes: + index_score[b,s,t] = Σ_h (weight[b,s,h] · softmax_scale · q[b,s,h,:] · k[b,t,:]) + + Args: + hidden_states: Input hidden states `[B, S, hidden_size]`. + q_resid: Query residual from `q_a_layernorm(q_a_proj(x))`, shape `[B, S, q_lora_rank]`. + position_embeddings: `(cos, sin)` from RotaryEmbedding. + attention_mask: Causal mask, broadcastable to `[B, S, T]`. + use_cache: Whether to store/update the indexer's own key cache for autoregressive decode. + + Returns: + `torch.LongTensor`: Top-k token indices of shape `[B, S, topk]`. + """ + batch_size, seq_len, _ = hidden_states.shape + cos, sin = position_embeddings + + # === Queries === + q = self.wq_b(q_resid) # [B, S, H*D] + q = q.view(batch_size, seq_len, self.n_heads, self.head_dim) # [B, S, H, D] + q_pe, q_nope = torch.split( + q, [self.qk_rope_head_dim, self.head_dim - self.qk_rope_head_dim], dim=-1 + ) + q_pe = apply_rotary_pos_emb( + q_pe, cos, sin, unsqueeze_dim=2 + ) # [B, S, H, rope_D] + q = torch.cat([q_pe, q_nope], dim=-1) # [B, S, H, D] + + # === Keys === + k = self.k_norm(self.wk(hidden_states)) # [B, S, D] + k_pe, k_nope = torch.split( + k, [self.qk_rope_head_dim, self.head_dim - self.qk_rope_head_dim], dim=-1 + ) + k_pe = apply_rotary_pos_emb( + k_pe.unsqueeze(2), cos, sin, unsqueeze_dim=2 + ).squeeze(2) # [B, S, rope_D] + k = torch.cat([k_pe, k_nope], dim=-1) # [B, S, D] + + # === Key cache (managed by the indexer, not DynamicCache) === + # Reset cache on prefill (new prompt) to avoid stale keys / batch-size mismatch + if seq_len > 1: + self._cached_keys = None + + if use_cache: + if self._cached_keys is not None: + k_cached = torch.cat([self._cached_keys, k], dim=1) # [B, T, D] + else: + k_cached = k + self._cached_keys = k_cached + else: + k_cached = k + + # === Scoring === + # Reference: weights = weights_proj(x.float()) * n_heads^(-0.5) + # Reference: weights = weights.unsqueeze(-1) * q_scale * softmax_scale + # Reference: index_score = fp8_index(q_fp8, weights, k_cache, k_scale_cache) + # + # In bf16 mode (no FP8), q_scale = 1. The fp8_index kernel computes: + # score[b,s,t] = sum_h(weights[b,s,h] * dot(q[b,s,h,:], k[b,t,:])) + # where weights already absorbs n_heads^(-0.5) and softmax_scale. + + # Don't force fp32 inputs here: the checkpoint stores `weights_proj.weight` in bf16. + # Use native dtype for matmul, then upcast the result for scoring stability. + weights = self.weights_proj(hidden_states).float() * ( + self.n_heads**-0.5 + ) # [B, S, H] + + # q·k^T per head: [B, S, H, D] @ [B, T, D]^T → [B, S, H, T] + scores = ( + torch.einsum("bshd,btd->bsht", q.float(), k_cached.float()) + * self.softmax_scale + ) + + # Weight per head and sum across heads → [B, S, T] + index_scores = torch.einsum("bsht,bsh->bst", scores, weights) + + if attention_mask is not None: + index_scores = index_scores + attention_mask + + total_len = index_scores.shape[-1] + topk = min(self.index_topk, total_len) + topk_indices = index_scores.topk(topk, dim=-1).indices # [B, S, topk] + return topk_indices + + +class GlmMoeDsaAttention(nn.Module): + """ + Multi-head Latent Attention (MLA) with Dynamic Sparse Attention (DSA) indexer. + + This follows the same architecture as DeepSeek V3.2's MLA: + - Query: x → q_a_proj → RMSNorm → q_b_proj → split(q_nope, q_pe) → RoPE(q_pe) + - KV: x → kv_a_proj → split(kv_compressed, k_pe) → RMSNorm(kv_compressed) → kv_b_proj + → RoPE(k_pe) + - Cache: fully expanded key_states [B, H, T, qk_head_dim] and value_states [B, H, T, v_head_dim] + - Indexer: selects top-k tokens via DSA, applied as an additive -inf mask on attention scores + + **Caching strategy**: follows the DeepSeek V3 transformers convention of fully expanding K/V + before caching. This ensures compatibility with DynamicCache, StaticCache, flash attention, + and SDPA backends. The reference's compressed-cache decode path (which avoids the kv_b_proj + expansion at decode time) is a future optimization that would require a dedicated MLA cache class. + + **FP8 compatibility**: all weight accesses use standard nn.Linear forward calls (never + raw `.weight` access), so FP8-quantized checkpoints work transparently. + """ + + def __init__(self, config: GlmMoeDsaConfig, layer_idx: int): + super().__init__() + self.config = config + self.layer_idx = layer_idx + self.num_key_value_groups = ( + config.num_attention_heads // config.num_key_value_heads + ) + self.attention_dropout = config.attention_dropout + self.num_heads = config.num_attention_heads + + self.q_lora_rank = config.q_lora_rank + self.qk_rope_head_dim = config.qk_rope_head_dim + self.kv_lora_rank = config.kv_lora_rank + self.v_head_dim = config.v_head_dim + self.qk_nope_head_dim = config.qk_nope_head_dim + self.qk_head_dim = config.qk_head_dim + + self.is_causal = True + + # Query projection (with optional LoRA) + if self.q_lora_rank is None: + self.q_proj = nn.Linear( + config.hidden_size, self.num_heads * self.qk_head_dim, bias=False + ) + else: + self.q_a_proj = nn.Linear( + config.hidden_size, config.q_lora_rank, bias=config.attention_bias + ) + self.q_a_layernorm = GlmMoeDsaRMSNorm(config.q_lora_rank) + self.q_b_proj = nn.Linear( + config.q_lora_rank, self.num_heads * self.qk_head_dim, bias=False + ) + + # Key-Value projections (MLA compressed path) + self.kv_a_proj_with_mqa = nn.Linear( + config.hidden_size, + self.kv_lora_rank + self.qk_rope_head_dim, + bias=config.attention_bias, + ) + self.kv_a_layernorm = GlmMoeDsaRMSNorm(self.kv_lora_rank) + self.kv_b_proj = nn.Linear( + self.kv_lora_rank, + self.num_heads * (self.qk_nope_head_dim + self.v_head_dim), + bias=False, + ) + + # Output projection + self.o_proj = nn.Linear( + self.num_heads * self.v_head_dim, + config.hidden_size, + bias=config.attention_bias, + ) + + self.scaling = self.qk_head_dim ** (-0.5) + + self.indexer = GlmMoeDsaIndexer(config, layer_idx) + + def forward( + self, + hidden_states: torch.Tensor, + position_embeddings: tuple[torch.Tensor, torch.Tensor], + attention_mask: torch.Tensor | None, + past_key_values: Cache | None = None, + cache_position: torch.LongTensor | None = None, + **kwargs: Unpack[FlashAttentionKwargs], + ) -> tuple[torch.Tensor, torch.Tensor | None, tuple[torch.Tensor] | None]: + batch_size, seq_length = hidden_states.shape[:-1] + cos, sin = position_embeddings + + # ===== Query path ===== + if self.q_lora_rank is None: + query_states = self.q_proj(hidden_states) + q_resid = None + else: + q_resid = self.q_a_layernorm( + self.q_a_proj(hidden_states) + ) # [B, S, q_lora_rank] + query_states = self.q_b_proj(q_resid) + query_states = query_states.view( + batch_size, seq_length, self.num_heads, self.qk_head_dim + ).transpose(1, 2) + # Split nope/rope, apply RoPE, recombine — layout: [B, H, S, D] + q_nope, q_pe = torch.split( + query_states, [self.qk_nope_head_dim, self.qk_rope_head_dim], dim=-1 + ) + q_pe = apply_rotary_pos_emb(q_pe, cos, sin, unsqueeze_dim=1) # BHSD format + + # ===== KV path ===== + compressed_kv = self.kv_a_proj_with_mqa( + hidden_states + ) # [B, S, kv_rank + rope_D] + k_compressed, k_pe = torch.split( + compressed_kv, [self.kv_lora_rank, self.qk_rope_head_dim], dim=-1 + ) + k_compressed = self.kv_a_layernorm(k_compressed) # [B, S, kv_rank] + + # Expand KV through kv_b_proj + kv_expanded = self.kv_b_proj(k_compressed) # [B, S, H * (nope_D + v_D)] + kv_expanded = kv_expanded.view( + batch_size, + seq_length, + self.num_heads, + self.qk_nope_head_dim + self.v_head_dim, + ) + k_nope, value_states = torch.split( + kv_expanded, [self.qk_nope_head_dim, self.v_head_dim], dim=-1 + ) + k_nope = k_nope.transpose(1, 2) # [B, H, S, nope_D] + value_states = value_states.transpose(1, 2) # [B, H, S, v_D] + + # RoPE on k_pe (single-head rope stream) + k_pe = k_pe.view( + batch_size, 1, seq_length, self.qk_rope_head_dim + ) # [B, 1, S, rope_D] + k_pe = apply_rotary_pos_emb(k_pe, cos, sin, unsqueeze + _dim=1) # BHSD format + k_pe = k_pe.expand(-1, self.num_heads, -1, -1) # [B, H, S, rope_D] + + # Assemble full Q and K + query_states = torch.cat([q_nope, q_pe], dim=-1) # [B, H, S, qk_head_dim] + key_states = torch.cat([k_nope, k_pe], dim=-1) # [B, H, S, qk_head_dim] + + # Cache update + if past_key_values is not None: + cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} + key_states, value_states = past_key_values.update( + key_states, value_states, self.layer_idx, cache_kwargs + ) + + # ===== Indexer (DSA sparse mask) ===== + # attention_mask is [B, 1, S, T] (4D) for eager and (2D) otherwise but indexer works with [B, S, T] (3D) + indexer_mask = ( + attention_mask[:, 0, :, :] + if attention_mask is not None and attention_mask.dim() == 4 + else attention_mask.unsqueeze(1) + if attention_mask is not None + else None + ) + topk_indices = self.indexer( + hidden_states, + q_resid, + position_embeddings, + indexer_mask, + use_cache=past_key_values is not None, + ) # [B, S, topk] + + # Build combined DSA + causal mask: -inf everywhere except selected top-k positions + total_len = key_states.shape[2] + index_mask = torch.full( + (batch_size, seq_length, total_len), + float("-inf"), + device=hidden_states.device, + dtype=query_states.dtype, + ) + index_mask.scatter_(-1, topk_indices, 0.0) # [B, S, T] + index_mask = index_mask.unsqueeze(1) # [B, 1, S, T] + if attention_mask is not None and attention_mask.dim() == 4: + causal_mask = attention_mask[..., :total_len] + combined_mask = index_mask + causal_mask + else: + combined_mask = ( + attention_mask.masked_fill(index_mask == float("-inf"), float("-inf")) + if attention_mask is not None + else index_mask + ) + + # Flash attention head_dim padding (qk_head_dim != v_head_dim) + if ( + is_flash_attention_requested(self.config) + and self.qk_head_dim != self.v_head_dim + ): + value_states = F.pad(value_states, [0, self.qk_head_dim - self.v_head_dim]) + + attention_interface: Callable = ALL_ATTENTION_FUNCTIONS.get_interface( + self.config._attn_implementation, eager_attention_forward + ) + + attn_output, attn_weights = attention_interface( + self, + query_states, + key_states, + value_states, + combined_mask, + dropout=0.0 if not self.training else self.attention_dropout, + scaling=self.scaling, + indices=topk_indices, # flash_mla_with_kvcache + **kwargs, + ) + + if ( + is_flash_attention_requested(self.config) + and self.qk_head_dim != self.v_head_dim + ): + attn_output = attn_output[:, :, :, : self.v_head_dim] + + attn_output = attn_output.reshape(batch_size, seq_length, -1).contiguous() + attn_output = self.o_proj(attn_output) + return attn_output, attn_weights + + +class GlmMoeDsaDecoderLayer(Glm4MoeLiteDecoderLayer): + pass + + +class GlmMoeDsaPreTrainedModel(Glm4MoePreTrainedModel): + # NOTE: FP8 quantization uses `_keep_in_fp32_modules` (not `_strict`) to decide which modules to NOT convert. + # We must keep `indexer.weights_proj` as a plain Linear to match the checkpoint (no `weight_scale_inv`). + _keep_in_fp32_modules = ["indexer.weights_proj"] + _keep_in_fp32_modules_strict = ["e_score_correction_bias"] + _keys_to_ignore_on_load_unexpected = [r"model\.layers\.78.*"] + _supports_flash_attn = ( + False # flash-mla kernels need a bit more work in the way we enable them! + ) + _supports_sdpa = True + _supports_flex_attn = False + _default_flash_implementation = "kernels-community/flash-mla" + + +class GlmMoeDsaModel(Glm4MoeModel): + pass + + +class GlmMoeDsaForCausalLM(Glm4MoeForCausalLM): + pass + + +__all__ = [ + "GlmMoeDsaConfig", + "GlmMoeDsaPreTrainedModel", + "GlmMoeDsaModel", + "GlmMoeDsaForCausalLM", +]