Public tokenizer errors, hasChatTemplate (#171)

* Make tokenizer errors public, add hasChatTemplate

* Tests, better message.

Author: pcuenca

Sources/Tokenizers/Tokenizer.swift

@@ -12,7 +12,7 @@ import Jinja
 public typealias Message = [String: Any]
 public typealias ToolSpec = [String: Any]
 
-enum TokenizerError: Error {
+public enum TokenizerError: Error {
     case missingConfig
     case missingTokenizerClassInConfig
     case unsupportedTokenizer(String)
@@ -144,6 +144,8 @@ public protocol Tokenizer {
     var unknownToken: String? { get }
     var unknownTokenId: Int? { get }
 
+    var hasChatTemplate: Bool { get }
+
     /// The appropriate chat template is selected from the tokenizer config
     func applyChatTemplate(messages: [Message]) throws -> [Int]
 
@@ -182,6 +184,8 @@ public protocol Tokenizer {
 }
 
 extension Tokenizer {
+    public var hasChatTemplate: Bool { false }
+
     /// Call previous signature for backwards compatibility
     func applyChatTemplate(
         messages: [Message],
@@ -399,6 +403,10 @@ public class PreTrainedTokenizer: Tokenizer {
         model.convertIdToToken(id)
     }
 
+    public var hasChatTemplate: Bool {
+        return tokenizerConfig.chatTemplate != nil
+    }
+
     public func applyChatTemplate(messages: [Message]) throws -> [Int] {
         try applyChatTemplate(messages: messages, addGenerationPrompt: true)
     }
@@ -485,7 +493,7 @@ public class PreTrainedTokenizer: Tokenizer {
         }
 
         guard let selectedChatTemplate else {
-            throw TokenizerError.chatTemplate("No chat template was specified")
+            throw TokenizerError.chatTemplate("This tokenizer does not have a chat template, and no template was passed.")
         }
 
         let template = try Template(selectedChatTemplate)

Tests/TokenizersTests/ChatTemplateTests.swift

@@ -169,4 +169,26 @@ What is the weather in Paris today?<|im_end|>
         XCTAssertTrue(decoded.hasPrefix(expectedPromptStart), "Prompt should start with expected system message")
         XCTAssertTrue(decoded.hasSuffix(expectedPromptEnd), "Prompt should end with expected format")
     }
+
+    func testHasChatTemplate() async throws {
+        var tokenizer = try await AutoTokenizer.from(pretrained: "google-bert/bert-base-uncased")
+        XCTAssertFalse(tokenizer.hasChatTemplate)
+
+        tokenizer = try await AutoTokenizer.from(pretrained: "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B")
+        XCTAssertTrue(tokenizer.hasChatTemplate)
+    }
+
+    func testApplyTemplateError() async throws {
+        let tokenizer = try await AutoTokenizer.from(pretrained: "google-bert/bert-base-uncased")
+        XCTAssertFalse(tokenizer.hasChatTemplate)
+        XCTAssertThrowsError(try tokenizer.applyChatTemplate(messages: []))
+        do {
+            _ = try tokenizer.applyChatTemplate(messages: [])
+            XCTFail()
+        } catch TokenizerError.chatTemplate(let message) {
+            XCTAssertEqual(message, "This tokenizer does not have a chat template, and no template was passed.")
+        } catch {
+            XCTFail()
+        }
+    }
 }