Update src/transformers/models/llama4/configuration_llama4.py

ArthurZucker · web-flow · commit b077bb5b19b2 · 2025-04-01T10:30:41.000+02:00
diff --git a/src/transformers/models/llama4/configuration_llama4.py b/src/transformers/models/llama4/configuration_llama4.py
@@ -142,8 +142,6 @@ class Llama4TextConfig(PretrainedConfig):
         "layers.*.feed_forward.experts.gate_up_proj": "local_packed_rowwise",  # row because not linear
         "layers.*.feed_forward.experts.down_proj": "local_colwise",  # col because not linear
         "layers.*.feed_forward.experts": "local",
-        "layers.*.feed_forward.shared_expert.gate_proj": "local_colwise",
-        "layers.*.feed_forward.shared_expert.up_proj": "local_colwise",
         "layers.*.feed_forward.down_proj": "local_rowwise",
         "layers.*.feed_forward": "gather",
     }

Original file line number	Diff line number	Diff line change
`@@ -142,8 +142,6 @@ class Llama4TextConfig(PretrainedConfig):`
`142`	`142`	`"layers.*.feed_forward.experts.gate_up_proj": "local_packed_rowwise", # row because not linear`
`143`	`143`	`"layers.*.feed_forward.experts.down_proj": "local_colwise", # col because not linear`
`144`	`144`	`"layers.*.feed_forward.experts": "local",`
`145`		`- "layers.*.feed_forward.shared_expert.gate_proj": "local_colwise",`
`146`		`- "layers.*.feed_forward.shared_expert.up_proj": "local_colwise",`
`147`	`145`	`"layers.*.feed_forward.down_proj": "local_rowwise",`
`148`	`146`	`"layers.*.feed_forward": "gather",`
`149`	`147`	`}`