Fix messages format for text-only chats (#207)

Author: DePasqualeOrg

Applications/VLMEval/ContentView.swift

@@ -395,25 +395,34 @@ class VLMEvaluator {
                     } else {
                         []
                     }
-                var userInput = UserInput(
-                    messages: [
+                let messages: [[String: Any]] =
+                    if !images.isEmpty || !videos.isEmpty {
                         [
-                            "role": "user",
-                            "content": [
-                                ["type": "text", "text": prompt]
+                            [
+                                "role": "user",
+                                "content": [
+                                    ["type": "text", "text": prompt]
+                                ]
+                                    // Messages format for Qwen 2 VL, Qwen 2.5 VL. May need to be adapted for other models.
+                                    + images.map { _ in
+                                        ["type": "image"]
+                                    }
+                                    + videos.map { _ in
+                                        ["type": "video"]
+                                    },
                             ]
-                                + images.map { _ in
-                                    ["type": "image"]
-                                }
-                                + videos.map { _ in
-                                    ["type": "video"]
-                                },
                         ]
-                    ], images: images, videos: videos)
+                    } else {
+                        [
+                            [
+                                "role": "user",
+                                "content": prompt,
+                            ]
+                        ]
+                    }
+                var userInput = UserInput(messages: messages, images: images, videos: videos)
                 userInput.processing.resize = .init(width: 448, height: 448)
-
                 let input = try await context.processor.prepare(input: userInput)
-
                 return try MLXLMCommon.generate(
                     input: input,
                     parameters: generateParameters,

Tools/llm-tool/LLMTool.swift

@@ -216,28 +216,30 @@ struct EvaluateCommand: AsyncParsableCommand {
         let prompt =
             (try? generate.resolvePrompt(configuration: modelConfiguration))
             ?? modelConfiguration.defaultPrompt
-
         let images = image.map { UserInput.Image.url($0) }
         let videos = video.map { UserInput.Video.url($0) }
-
-        let messages: [[String: Any]] = [
-            [
-                "role": "user",
-                "content": [
-                    ["type": "text", "text": prompt]
+        let messages: [[String: Any]] =
+            if !images.isEmpty || !videos.isEmpty {
+                [
+                    [
+                        "role": "user",
+                        "content": [
+                            ["type": "text", "text": prompt]
+                        ]
+                            // Messages format for Qwen 2 VL, Qwen 2.5 VL. May need to be adapted for other models.
+                            + images.map { _ in ["type": "image"] }
+                            + videos.map { _ in ["type": "video"] },
+                    ]
                 ]
-                    // Messages format for Qwen 2 VL, Qwen 2.5 VL. May need to be adapted for other models.
-                    + images.map { _ in ["type": "image"] }
-                    + videos.map { _ in ["type": "video"] },
-            ]
-        ]
-
-        var input = UserInput(
-            messages: messages,
-            images: images,
-            videos: videos
-        )
-
+            } else {
+                [
+                    [
+                        "role": "user",
+                        "content": prompt,
+                    ]
+                ]
+            }
+        var userInput = UserInput(messages: messages, images: images, videos: videos)
         if !resize.isEmpty {
             let size: CGSize
             if resize.count == 1 {
@@ -249,10 +251,9 @@ struct EvaluateCommand: AsyncParsableCommand {
                 let v1 = resize[1]
                 size = CGSize(width: v0, height: v1)
             }
-            input.processing.resize = size
+            userInput.processing.resize = size
         }
-
-        return input
+        return userInput
     }
 
     @MainActor

mlx-swift-examples.xcodeproj/xcshareddata/xcschemes/llm-tool.xcscheme

@@ -55,6 +55,14 @@
             argument = "--model mlx-community/CodeLlama-13b-Instruct-hf-4bit-MLX"
             isEnabled = "NO">
          </CommandLineArgument>
+         <CommandLineArgument
+            argument = "--prompt &apos;Describe the image in English.&apos; --image https://www.gstatic.com/webp/gallery/1.webp"
+            isEnabled = "NO">
+         </CommandLineArgument>
+         <CommandLineArgument
+            argument = "--model mlx-community/Qwen2-VL-2B-Instruct-4bit"
+            isEnabled = "NO">
+         </CommandLineArgument>
          <CommandLineArgument
             argument = "--repetition-penalty 1.2"
             isEnabled = "NO">
@@ -79,18 +87,18 @@
             argument = "--prompt &apos;def quick_sort(arr, left=None, right=None):&apos;"
             isEnabled = "NO">
          </CommandLineArgument>
+         <CommandLineArgument
+            argument = "--prompt &apos;Why is the sky blue?&apos;"
+            isEnabled = "YES">
+         </CommandLineArgument>
          <CommandLineArgument
             argument = "--model mlx-community/Mistral-7B-v0.1-hf-4bit-mlx"
             isEnabled = "NO">
          </CommandLineArgument>
          <CommandLineArgument
-            argument = "--prompt &apos;Describe the image in English.&apos; --image https://www.gstatic.com/webp/gallery/1.webp"
+            argument = "--model mlx-community/Llama-3.2-1B-Instruct-4bit"
             isEnabled = "YES">
          </CommandLineArgument>
-         <CommandLineArgument
-            argument = "--model mlx-community/quantized-gemma-2b-it"
-            isEnabled = "NO">
-         </CommandLineArgument>
          <CommandLineArgument
             argument = "--model mlx-community/phi-2-hf-4bit-mlx"
             isEnabled = "NO">