Fix MoE vs FF (#41)

pcuenca · web-flow · commit a43e0561a4a1 · 2025-04-04T20:08:42.000+02:00
diff --git a/src/transformers/models/llama4/configuration_llama4.py b/src/transformers/models/llama4/configuration_llama4.py
@@ -236,7 +236,7 @@ def __init__(
 
         self.interleave_moe_layer_step = interleave_moe_layer_step
         self.moe_layers = (
-            moe_layers if moe_layers is not None else list(range(0, num_hidden_layers, interleave_moe_layer_step))
+            moe_layers if moe_layers is not None else list(range(interleave_moe_layer_step-1, num_hidden_layers, interleave_moe_layer_step))
         )
         self.attention_chunk_size = attention_chunk_size
 

Original file line number	Diff line number	Diff line change
`@@ -236,7 +236,7 @@ def __init__(`
`236`	`236`
`237`	`237`	`self.interleave_moe_layer_step = interleave_moe_layer_step`
`238`	`238`	`self.moe_layers = (`
`239`		`- moe_layers if moe_layers is not None else list(range(0, num_hidden_layers, interleave_moe_layer_step))`
	`239`	`+ moe_layers if moe_layers is not None else list(range(interleave_moe_layer_step-1, num_hidden_layers, interleave_moe_layer_step))`
`240`	`240`	`)`
`241`	`241`	`self.attention_chunk_size = attention_chunk_size`
`242`	`242`