Add llm-models function, to list available models for a service (#165)

Author: ahyatt

NEWS.org

@@ -1,7 +1,8 @@
 * Version 0.24.0
 - Add =multi-output= as an option, allowing all llm results to return, call, or stream multiple kinds of data via a plist.  This allows separating out reasoning, as well as optionally returning text as well as tool uses at the same time.
+- Added ~llm-models~ to get a list of models from a provider.
 - Fix misnamed ~llm-capabilities~ output to refer to =tool-use= and =streaming-tool-use= (which is new).
-- Fixed Clude streaming tool use (via Paul Nelson)
+- Fixed Claude streaming tool use (via Paul Nelson)
 - Added Deepseek service
 - Add Gemini 2.0 pro experimental model, default to 2.0 flash
 - Add Open AI's o3 mini model

README.org

@@ -154,6 +154,7 @@ For all callbacks, the callback will be executed in the buffer the function was
 - ~llm-count-tokens provider string~: Count how many tokens are in ~string~.  This may vary by ~provider~, because some provideres implement an API for this, but typically is always about the same.  This gives an estimate if the provider has no API support.
 - ~llm-cancel-request request~ Cancels the given request, if possible.  The ~request~ object is the return value of async and streaming functions.
 - ~llm-name provider~.  Provides a short name of the model or provider, suitable for showing to users.
+- ~llm-models provider~.  Return a list of all the available model names for the provider.  This could be either embedding or chat models.  You can use ~llm-models-match~ to filter on models that have a certain capability (as long as they are in ~llm-models~).
 - ~llm-chat-token-limit~.  Gets the token limit for the chat model.  This isn't possible for some backends like =llama.cpp=, in which the model isn't selected or known by this library.
 
   And the following helper functions:

llm-claude.el

@@ -28,6 +28,7 @@
 (require 'llm)
 (require 'llm-provider-utils)
 (require 'plz-event-source)
+(require 'plz)
 (require 'rx)
 
 ;; Models defined at https://docs.anthropic.com/claude/docs/models-overview
@@ -226,9 +227,13 @@ DATA is a vector of lists produced by `llm-provider-streaming-media-handler'."
     (format "Error %s: '%s'" (assoc-default 'type err)
             (assoc-default 'message err))))
 
+(defun llm-claude--url (method)
+  "Return a Claude API URL for METHOD."
+  (format "https://api.anthropic.com/v1/%s" method))
+
 (cl-defmethod llm-provider-chat-url ((_ llm-claude))
   "Return the URL for the Claude API."
-  "https://api.anthropic.com/v1/messages")
+  (llm-claude--url "messages"))
 
 (cl-defmethod llm-chat-token-limit ((provider llm-claude))
   (llm-provider-utils-model-token-limit (llm-claude-chat-model provider)))
@@ -249,6 +254,15 @@ DATA is a vector of lists produced by `llm-provider-streaming-media-handler'."
                                                                           'user
                                                                         'assistant)))
 
+(cl-defmethod llm-models ((provider llm-claude))
+  (mapcar (lambda (model)
+            (plist-get model :id))
+          (append
+           (plist-get (plz 'get (llm-claude--url "models")
+                        :as (lambda () (json-parse-buffer :object-type 'plist))
+                        :headers (llm-provider-headers provider))
+                      :data)
+           nil)))
 
 (provide 'llm-claude)
 

llm-deepseek.el

@@ -70,7 +70,7 @@ RESPONSE can be nil if the response is complete."
                              (funcall receiver response))))))))))
 
 (cl-defmethod llm-capabilities ((provider llm-deepseek))
-  (append '(streaming)
+  (append '(streaming model-list)
           (when-let* ((model (llm-models-match (llm-deepseek-chat-model provider))))
             (llm-model-capabilities model))))
 

llm-gemini.el

@@ -58,11 +58,14 @@ You can get this at https://makersuite.google.com/app/apikey."
 (cl-defmethod llm-provider-embedding-extract-result ((_ llm-gemini) response)
   (assoc-default 'values (assoc-default 'embedding response)))
 
+(defconst llm-gemini--base-url "https://generativelanguage.googleapis.com/v1beta/models/")
+
 ;; from https://ai.google.dev/tutorials/rest_quickstart
 (defun llm-gemini--chat-url (provider streaming-p)
   "Return the URL for the chat request, using PROVIDER.
 If STREAMING-P is non-nil, use the streaming endpoint."
-  (format "https://generativelanguage.googleapis.com/v1beta/models/%s:%s?key=%s"
+  (format "%s%s:%s?key=%s"
+          llm-gemini--base-url
           (llm-gemini-chat-model provider)
           (if streaming-p "streamGenerateContent" "generateContent")
           (if (functionp (llm-gemini-key provider))
@@ -90,13 +93,25 @@ If STREAMING-P is non-nil, use the streaming endpoint."
 
 (cl-defmethod llm-capabilities ((provider llm-gemini))
   (append
-   (list 'streaming 'embeddings)
+   '(streaming embeddings model-list)
    (when-let ((model (llm-models-match (llm-gemini-chat-model provider)))
               (capabilities (llm-model-capabilities model)))
      (append
       (when (member 'tool-use capabilities) '(tool-use streaming-tool-use))
       (seq-intersection capabilities '(image-input audio-input video-input))))))
 
+(cl-defmethod llm-models ((provider llm-gemini))
+  (mapcar (lambda (model)
+            (plist-get model :name))
+          (append
+           (plist-get (plz 'get (format "%s?key=%s" llm-gemini--base-url
+                                        (if (functionp (llm-gemini-key provider))
+                                            (funcall (llm-gemini-key provider))
+                                          (llm-gemini-key provider)))
+                        :as (lambda () (json-parse-buffer :object-type 'plist)))
+                      :models)
+           nil)))
+
 (provide 'llm-gemini)
 
 ;;; llm-gemini.el ends here

llm-integration-test.el

@@ -422,4 +422,11 @@ else.  We really just want to see if it's in the right ballpark."
     (should (integerp result))
     (should (> result 0))))
 
+(llm-def-integration-test llm-models (provider)
+  (when (member 'model-list (llm-capabilities provider))
+    (let ((models (llm-models provider)))
+      (should models)
+      (should (listp models))
+      (should (> (length models) 0)))))
+
 (provide 'llm-integration-test)

llm-ollama.el

@@ -29,6 +29,7 @@
 (require 'llm)
 (require 'llm-provider-utils)
 (require 'llm-models)
+(require 'plz)
 (require 'json)
 (require 'plz-media-type)
 
@@ -229,7 +230,7 @@ PROVIDER is the llm-ollama provider."
                                         2048))
 
 (cl-defmethod llm-capabilities ((provider llm-ollama))
-  (append '(streaming json-response)
+  (append '(streaming json-response model-list)
           (when (and (llm-ollama-embedding-model provider)
                      (let ((embedding-model (llm-models-match
                                              (llm-ollama-embedding-model provider))))
@@ -243,6 +244,14 @@ PROVIDER is the llm-ollama provider."
              (when (member 'tool-use capabilities) '(tool-use))
              (seq-intersection capabilities '(image-input))))))
 
+(cl-defmethod llm-models ((provider llm-ollama))
+  (mapcar (lambda (model-data)
+            (plist-get model-data :name))
+          (plist-get (plz 'get (llm-ollama--url provider "tags")
+                       :as (lambda ()
+                             (json-parse-buffer :object-type 'plist)))
+                     :models )))
+
 (provide 'llm-ollama)
 
 ;;; llm-ollama.el ends here

llm-openai.el

@@ -30,6 +30,7 @@
 (require 'llm-provider-utils)
 (require 'llm-models)
 (require 'json)
+(require 'plz)
 (require 'plz-event-source)
 
 (defgroup llm-openai nil
@@ -369,18 +370,28 @@ RESPONSE can be nil if the response is complete."
   (llm-provider-utils-model-token-limit (llm-openai-chat-model provider)))
 
 (cl-defmethod llm-capabilities ((provider llm-openai))
-  (append '(streaming embeddings tool-use streaming-tool-use json-response)
+  (append '(streaming embeddings tool-use streaming-tool-use json-response model-list)
           (when-let ((model (llm-models-match (llm-openai-chat-model provider))))
             (seq-intersection (llm-model-capabilities model)
                               '(image-input)))))
 
 (cl-defmethod llm-capabilities ((provider llm-openai-compatible))
-  (append '(streaming)
+  (append '(streaming model-list)
           (when (llm-openai-embedding-model provider)
             '(embeddings embeddings-batch))
           (when-let* ((model (llm-models-match (llm-openai-chat-model provider))))
             (llm-model-capabilities model))))
 
+(cl-defmethod llm-models ((provider llm-openai))
+  (mapcar (lambda (model)
+            (plist-get model :id))
+          (append
+           (plist-get (plz 'get (llm-openai--url provider "models")
+                        :as (lambda () (json-parse-buffer :object-type 'plist))
+                        :headers (llm-openai--headers provider))
+                      :data)
+           nil)))
+
 (provide 'llm-openai)
 
 ;;; llm-openai.el ends here

llm.el

@@ -491,11 +491,7 @@ be passed to `llm-cancel-request'."
   "Log the input to llm-chat-async."
   (llm--log 'api-send :provider provider :prompt prompt)
   ;; We need to wrap the callbacks before we set llm-log to nil.
-  (let* ((new-partial-callback (lambda (response)
-                                 (when partial-callback
-                                   (let ((llm-log nil))
-                                     (funcall partial-callback response)))))
-         (new-response-callback (lambda (response)
+  (let* ((new-response-callback (lambda (response)
                                   (llm--log 'api-receive :provider provider :msg response)
                                   (let ((llm-log nil))
                                     (funcall response-callback response))))
@@ -505,7 +501,7 @@ be passed to `llm-cancel-request'."
                                (let ((llm-log nil))
                                  (funcall error-callback type err))))
          (llm-log nil)
-         (result (cl-call-next-method provider prompt new-partial-callback
+         (result (cl-call-next-method provider prompt partial-callback
                                       new-response-callback
                                       new-error-callback multi-output)))
     result))
@@ -593,7 +589,9 @@ JSON format.
 
 `video-input': the LLM can accept video as input.
 
-`audio-input': the LLM can accept audio as input."
+`audio-input': the LLM can accept audio as input.
+
+`model-list': the provider can return a list of models."
   (ignore provider)
   nil)
 
@@ -705,6 +703,15 @@ methods."
 (cl-defmethod llm-cancel-request ((proc process))
   (delete-process proc))
 
+(cl-defgeneric llm-models (provider)
+  "Return a list of model names for PROVIDER.
+This is not asynchronous, but should be fast.
+
+Not every model provides this, you can check the ones that implement
+`model-list' in `llm-capabilities' before calling."
+  (ignore provider)
+  (signal 'not-implemented nil))
+
 (cl-defgeneric llm-name (_)
   "Return the name of the model in PROVIDER.
 This is expected to be suitable for short labels.  For example, if