Skip to content

Commit 11b7b43

Browse files
authored
fix for #214 -- Qwen 2.5 fails to load: Key weight not found in Linear (#215)
- Qwen2 (LLM) had slightly incorrect logic in the initialization regarding lm_head - it was initialized even if not used, but this causes parameter loading to fail with current 0.21.3 mlx-swift
1 parent b23bbf0 commit 11b7b43

File tree

1 file changed

+8
-5
lines changed

1 file changed

+8
-5
lines changed

Libraries/MLXLLM/Models/Qwen2.swift

+8-5
Original file line numberDiff line numberDiff line change
@@ -172,22 +172,25 @@ public class Qwen2Model: Module, LLMModel, KVCacheDimensionProvider {
172172
private let model: Qwen2ModelInner
173173
let configuration: Qwen2Configuration
174174

175-
@ModuleInfo(key: "lm_head") var lmHead: Linear
175+
@ModuleInfo(key: "lm_head") var lmHead: Linear?
176176

177177
public init(_ args: Qwen2Configuration) {
178178
self.configuration = args
179179
self.vocabularySize = args.vocabularySize
180180
self.kvHeads = (0 ..< args.hiddenLayers).map { _ in args.kvHeads }
181181
self.model = Qwen2ModelInner(args)
182-
_lmHead.wrappedValue = Linear(args.hiddenSize, args.vocabularySize, bias: false)
182+
183+
if !args.tieWordEmbeddings {
184+
_lmHead.wrappedValue = Linear(args.hiddenSize, args.vocabularySize, bias: false)
185+
}
183186
}
184187

185188
public func callAsFunction(_ inputs: MLXArray, cache: [KVCache]?) -> MLXArray {
186189
var out = model(inputs, cache: cache)
187-
if configuration.tieWordEmbeddings {
188-
out = model.embedTokens.asLinear(out)
189-
} else {
190+
if let lmHead {
190191
out = lmHead(out)
192+
} else {
193+
out = model.embedTokens.asLinear(out)
191194
}
192195
return out
193196
}

0 commit comments

Comments
 (0)