Change function calling methods to tool-use, modify input format (#133)

This also adds a lot of testing, and also adds Gemini image
understanding.

Various bugs have been caught by test and fixed, including incorrect
temperatures in Gemini and Vertex, and incorrect use of maximum tokens
in Gemini.

Also, change version to 0.21.0, and update copyright years.

Author: ahyatt

NEWS.org

@@ -1,4 +1,7 @@
 * Version 0.21.0
+- Incompatible change to function calling, which is now tool use, affecting arguments and methods.
+- Support image understanding in Claude
+- Support streaming tool use in Claude
 - Add ~llm-models-add~ as a convenience method to add a model to the known list.
 * Version 0.20.0
 - Add ability to output according to a JSON spec.

README.org

@@ -1,6 +1,6 @@
 ;;; llm-azure.el --- llm module for integrating with Azure's Open AI service -*- lexical-binding: t; package-lint-main-file: "llm.el"; byte-compile-docstring-max-column: 200-*-
 
-;; Copyright (c) 2024  Free Software Foundation, Inc.
+;; Copyright (c) 2024-2025  Free Software Foundation, Inc.
 
 ;; Author: Andrew Hyatt <[email protected]>
 ;; Homepage: https://github.com/ahyatt/llm

llm-azure.el

@@ -1,6 +1,6 @@
 ;;; llm-claude.el --- llm module for integrating with Claude -*- lexical-binding: t; package-lint-main-file: "llm.el"; -*-
 
-;; Copyright (c) 2024  Free Software Foundation, Inc.
+;; Copyright (c) 2024-2025  Free Software Foundation, Inc.
 
 ;; Author: Andrew Hyatt <[email protected]>
 ;; Homepage: https://github.com/ahyatt/llm
@@ -43,64 +43,86 @@
   (unless (llm-claude-key provider)
     (error "No API key provided for Claude")))
 
-(defun llm-claude--tool-call (call)
-  "A Claude version of a function spec for CALL."
-  `(("name" . ,(llm-function-call-name call))
-    ("description" . ,(llm-function-call-description call))
-    ("input_schema" . ,(llm-provider-utils-openai-arguments
-                        (llm-function-call-args call)))))
+(defun llm-claude--tool-call (tool)
+  "A Claude version of a function spec for TOOL."
+  `(:name ,(llm-tool-function-name tool)
+          :description ,(llm-tool-function-description tool)
+          :input_schema ,(llm-provider-utils-openai-arguments
+                          (llm-tool-function-args tool))))
 
 (cl-defmethod llm-provider-chat-request ((provider llm-claude) prompt stream)
-  (let ((request `(("model" . ,(llm-claude-chat-model provider))
-                   ("stream" . ,(if stream t :json-false))
+  (let ((request
+          `(:model ,(llm-claude-chat-model provider)
+                   :stream ,(if stream t :false)
                    ;; Claude requires max_tokens
-                   ("max_tokens" . ,(or (llm-chat-prompt-max-tokens prompt) 4096))
-                   ("messages" .
-                    ,(mapcar (lambda (interaction)
-                               `(("role" . ,(pcase (llm-chat-prompt-interaction-role interaction)
-                                              ('function 'user)
-                                              ('assistant 'assistant)
-                                              ('user 'user)))
-                                 ("content" .
-                                  ,(if (llm-chat-prompt-interaction-function-call-results interaction)
-                                       (mapcar (lambda (result)
-                                                 `(("type" . "tool_result")
-                                                   ("tool_use_id" .
-                                                    ,(llm-chat-prompt-function-call-result-call-id
-                                                      result))
-                                                   ("content" .
-                                                    ,(llm-chat-prompt-function-call-result-result result))))
-                                               (llm-chat-prompt-interaction-function-call-results interaction))
-                                     (llm-chat-prompt-interaction-content interaction)))))
+                   :max_tokens ,(or (llm-chat-prompt-max-tokens prompt) 4096)
+                   :messages
+                   ,(vconcat
+                     (mapcar (lambda (interaction)
+                               `(:role  ,(pcase (llm-chat-prompt-interaction-role interaction)
+                                           ('tool_results "user")
+                                           ('tool_use "assistant")
+                                           ('assistant "assistant")
+                                           ('user "user"))
+                                        :content
+                                        ,(cond ((llm-chat-prompt-interaction-tool-results interaction)
+                                                (vconcat (mapcar (lambda (result)
+                                                                   `(:type "tool_result"
+                                                                           :tool_use_id
+                                                                           ,(llm-chat-prompt-tool-result-call-id result)
+                                                                           :content
+                                                                           ,(llm-chat-prompt-tool-result-result result)))
+                                                                 (llm-chat-prompt-interaction-tool-results interaction))))
+                                               ((llm-multipart-p (llm-chat-prompt-interaction-content interaction))
+                                                (llm-claude--multipart-content
+                                                 (llm-chat-prompt-interaction-content interaction)))
+                                               (t
+                                                (llm-chat-prompt-interaction-content interaction)))))
                              (llm-chat-prompt-interactions prompt)))))
         (system (llm-provider-utils-get-system-prompt prompt)))
-    (when (llm-chat-prompt-functions prompt)
-      (push `("tools" . ,(mapcar (lambda (f) (llm-claude--tool-call f))
-                                 (llm-chat-prompt-functions prompt))) request))
+    (when (llm-chat-prompt-tools prompt)
+      (plist-put request :tools
+                 (vconcat (mapcar (lambda (f) (llm-claude--tool-call f))
+                                  (llm-chat-prompt-tools prompt)))))
     (when (> (length system) 0)
-      (push `("system" . ,system) request))
+      (plist-put request :system system))
     (when (llm-chat-prompt-temperature prompt)
-      (push `("temperature" . ,(llm-chat-prompt-temperature prompt)) request))
-    (append request (llm-chat-prompt-non-standard-params prompt))))
-
-(cl-defmethod llm-provider-extract-function-calls ((_ llm-claude) response)
+      (plist-put request :temperature (llm-chat-prompt-temperature prompt)))
+    (append request (llm-provider-utils-non-standard-params-plist prompt))))
+
+(defun llm-claude--multipart-content (content)
+  "Return CONTENT as a list of Claude multipart content."
+  (vconcat (mapcar (lambda (part)
+                     (cond ((stringp part)
+                            `(:type "text"
+                                    :text ,part))
+                           ((llm-media-p part)
+                            `(:type "image"
+                                    :source (:type "base64"
+                                                   :media_type ,(llm-media-mime-type part)
+                                                   :data ,(base64-encode-string (llm-media-data part) t))))
+                           (t
+                            (error "Unsupported multipart content: %s" part))))
+                   (llm-multipart-parts content))))
+
+(cl-defmethod llm-provider-extract-tool-uses ((_ llm-claude) response)
   (let ((content (append (assoc-default 'content response) nil)))
     (cl-loop for item in content
              when (equal "tool_use" (assoc-default 'type item))
-             collect (make-llm-provider-utils-function-call
+             collect (make-llm-provider-utils-tool-use
                       :id (assoc-default 'id item)
                       :name (assoc-default 'name item)
                       :args (assoc-default 'input item)))))
 
-(cl-defmethod llm-provider-populate-function-calls ((_ llm-claude) prompt calls)
+(cl-defmethod llm-provider-populate-tool-uses ((_ llm-claude) prompt tool-uses)
   (llm-provider-utils-append-to-prompt
    prompt
-   (mapcar (lambda (call)
-             `((type . "tool_use")
-               (id . ,(llm-provider-utils-function-call-id call))
-               (name . ,(llm-provider-utils-function-call-name call))
-               (input . ,(llm-provider-utils-function-call-args call))))
-           calls)))
+   (vconcat (mapcar (lambda (call)
+                      `(:type "tool_use"
+                              :id ,(llm-provider-utils-tool-use-id call)
+                              :name ,(llm-provider-utils-tool-use-name call)
+                              :input ,(llm-provider-utils-tool-use-args call)))
+                    tool-uses))))
 
 (cl-defmethod llm-provider-chat-extract-result ((_ llm-claude) response)
   (let ((content (aref (assoc-default 'content response) 0)))
@@ -129,6 +151,9 @@
                          (when (equal type "text_delta")
                            (funcall msg-receiver (assoc-default 'text delta))))))))))
 
+(cl-defmethod llm-provider-collect-streaming-tool-uses ((_ llm-claude) data)
+  (llm-provider-utils-openai-collect-streaming-tool-uses data))
+
 (cl-defmethod llm-provider-headers ((provider llm-claude))
   `(("x-api-key" . ,(if (functionp (llm-claude-key provider))
                         (funcall (llm-claude-key provider))
@@ -153,16 +178,16 @@
   "Claude")
 
 (cl-defmethod llm-capabilities ((_ llm-claude))
-  (list 'streaming 'function-calls))
+  (list 'streaming 'function-calls 'image-input))
 
 (cl-defmethod llm-provider-append-to-prompt ((_ llm-claude) prompt result
-                                             &optional func-results)
+                                             &optional tool-use-results)
   ;; Claude doesn't have a 'function role, so we just always use assistant here.
   ;; But if it's a function result, it considers that a 'user response, which
   ;; needs to be sent back.
-  (llm-provider-utils-append-to-prompt prompt result func-results (if func-results
-                                                                      'user
-                                                                    'assistant)))
+  (llm-provider-utils-append-to-prompt prompt result tool-use-results (if tool-use-results
+                                                                          'user
+                                                                        'assistant)))
 
 
 (provide 'llm-claude)

llm-claude.el

@@ -1,6 +1,6 @@
 ;;; llm-fake.el --- Use for developers looking at llm calls. -*- lexical-binding: t; package-lint-main-file: "llm.el"; -*-
 
-;; Copyright (c) 2023, 2024  Free Software Foundation, Inc.
+;; Copyright (c) 2023-2025  Free Software Foundation, Inc.
 
 ;; Author: Andrew Hyatt <[email protected]>
 ;; Homepage: https://github.com/ahyatt/llm
@@ -31,7 +31,7 @@
 ;;; Code:
 
 (cl-defstruct llm-fake
- "A provider for the fake LLM provider.
+  "A provider for the fake LLM provider.
 
 OUTPUT-TO-BUFFER can be nil, in which case, nothing will be
 output.  If a string or a buffer, it will append the request as
@@ -44,7 +44,7 @@ message cons.  If nil, the response will be a short text string.
 EMBEDDING-ACTION-FUNC will be called with no arguments to produce
 either a vector response for the chat, or a signal symbol and
 message cons.  If nil, the response will be a simple vector."
- output-to-buffer chat-action-func embedding-action-func)
+  output-to-buffer chat-action-func embedding-action-func)
 
 (cl-defmethod llm-chat-async ((provider llm-fake) prompt response-callback error-callback)
   (condition-case err
@@ -104,9 +104,9 @@ message cons.  If nil, the response will be a simple vector."
       (let* ((f (llm-fake-embedding-action-func provider))
              (result (funcall f)))
         (pcase (type-of result)
-                ('vector result)
-                ('cons (signal (car result) (cdr result)))
-                (_ (error "Incorrect type found in `chat-embedding-func': %s" (type-of result)))))
+          ('vector result)
+          ('cons (signal (car result) (cdr result)))
+          (_ (error "Incorrect type found in `chat-embedding-func': %s" (type-of result)))))
     [0 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9]))
 
 (cl-defmethod llm-embedding-async ((provider llm-fake) string vector-callback error-callback)

llm-fake.el

@@ -1,6 +1,6 @@
 ;;; llm-gemini.el --- LLM implementation of Google Cloud Gemini AI -*- lexical-binding: t; package-lint-main-file: "llm.el"; -*-
 
-;; Copyright (c) 2023, 2024  Free Software Foundation, Inc.
+;; Copyright (c) 2023-2025  Free Software Foundation, Inc.
 
 ;; Author: Andrew Hyatt <[email protected]>
 ;; Homepage: https://github.com/ahyatt/llm
@@ -48,12 +48,12 @@ You can get this at https://makersuite.google.com/app/apikey."
   (format "https://generativelanguage.googleapis.com/v1beta/models/%s:embedContent?key=%s"
           (llm-gemini-embedding-model provider)
           (if (functionp (llm-gemini-key provider))
-              (funcall (llm-claude-key provider))
+              (funcall (llm-gemini-key provider))
             (llm-gemini-key provider))))
 
 (cl-defmethod llm-provider-embedding-request ((provider llm-gemini) string)
-  `((model . ,(llm-gemini-embedding-model provider))
-    (content . ((parts . (((text . ,string))))))))
+  `(:model ,(llm-gemini-embedding-model provider)
+           :content (:parts [(:text ,string)])))
 
 (cl-defmethod llm-provider-embedding-extract-result ((_ llm-gemini) response)
   (assoc-default 'values (assoc-default 'embedding response)))
@@ -75,22 +75,10 @@ If STREAMING-P is non-nil, use the streaming endpoint."
 (cl-defmethod llm-provider-chat-streaming-url ((provider llm-gemini))
   (llm-gemini--chat-url provider t))
 
-(cl-defmethod llm-provider-populate-function-calls ((_ llm-gemini) prompt calls)
-  (llm-provider-utils-append-to-prompt
-   prompt
-   ;; For Vertex there is just going to be one call
-   (mapcar (lambda (fc)
-             `((functionCall
-                .
-                ((name . ,(llm-provider-utils-function-call-name fc))
-                 (args . ,(llm-provider-utils-function-call-args fc))))))
-           calls)))
-
 (cl-defmethod llm-provider-chat-request ((_ llm-gemini) _ _)
-  (mapcar (lambda (c) (if (eq (car c) 'generation_config)
-                          (cons 'generationConfig (cdr c))
-                        c))
-          (cl-call-next-method)))
+  ;; Temporary, can be removed in the next version.  Without this the old
+  ;; definition will cause problems when users upgrade.
+  (cl-call-next-method))
 
 (cl-defmethod llm-name ((_ llm-gemini))
   "Return the name of PROVIDER."

llm-gemini.el

@@ -1,6 +1,6 @@
 ;;; llm-gpt4all.el --- llm module for integrating with GPT4All -*- lexical-binding: t; package-lint-main-file: "llm.el"; -*-
 
-;; Copyright (c) 2023, 2024  Free Software Foundation, Inc.
+;; Copyright (c) 2023-2025  Free Software Foundation, Inc.
 
 ;; Author: Andrew Hyatt <[email protected]>
 ;; Homepage: https://github.com/ahyatt/llm