Merge branch 'theroyallab:main' into main

Author: gittb

Diff for: .gitignore

@@ -192,7 +192,11 @@ templates/*
 !templates/place_your_templates_here.txt
 !templates/alpaca.jinja
 !templates/chatml.jinja
-!templates/chatml_with_headers_tool_calling.jinja
+
+# Tool calling templates folder
+templates/tool_calls/*
+!templates/tool_calls
+!templates/tool_calls/chatml_with_headers.jinja
 
 # Sampler overrides folder
 sampler_overrides/*

Diff for: backends/exllamav2/model.py

@@ -7,6 +7,7 @@
 import traceback
 import torch
 import uuid
+from copy import deepcopy
 from exllamav2 import (
     ExLlamaV2,
     ExLlamaV2Config,
@@ -400,19 +401,30 @@ def find_prompt_template(self, prompt_template_name, model_directory):
         find_template_functions = [
             lambda: PromptTemplate.from_model_json(
                 pathlib.Path(self.config.model_dir) / "tokenizer_config.json",
-                "chat_template",
+                key="chat_template",
             ),
             lambda: PromptTemplate.from_file(find_template_from_model(model_directory)),
         ]
 
+        # Find the template in the model directory if it exists
+        model_dir_template_path = (
+            pathlib.Path(self.config.model_dir) / "tabby_template.jinja"
+        )
+        if model_dir_template_path.exists():
+            find_template_functions[:0] = [
+                lambda: PromptTemplate.from_file(model_dir_template_path)
+            ]
+
         # Add lookup from prompt template name if provided
         if prompt_template_name:
             find_template_functions[:0] = [
-                lambda: PromptTemplate.from_file(prompt_template_name),
+                lambda: PromptTemplate.from_file(
+                    pathlib.Path("templates") / prompt_template_name
+                ),
                 lambda: PromptTemplate.from_model_json(
                     pathlib.Path(self.config.model_dir) / "tokenizer_config.json",
-                    "chat_template",
-                    prompt_template_name,
+                    key="chat_template",
+                    name=prompt_template_name,
                 ),
             ]
 
@@ -944,6 +956,14 @@ def check_unsupported_settings(self, **kwargs):
         Meant for dev wheels!
         """
 
+        if unwrap(kwargs.get("dry_allowed_length"), 0) > 0 and not hasattr(
+            ExLlamaV2Sampler.Settings, "dry_multiplier"
+        ):
+            logger.warning(
+                "DRY sampling is not supported by the currently "
+                "installed ExLlamaV2 version."
+            )
+
         return kwargs
 
     async def generate_gen(
@@ -1035,6 +1055,7 @@ async def generate_gen(
                     "Please use an ampere (30 series) or higher GPU for CFG support."
                 )
 
+        # Penalties
         gen_settings.token_repetition_penalty = unwrap(
             kwargs.get("repetition_penalty"), 1.0
         )
@@ -1070,6 +1091,32 @@ async def generate_gen(
             kwargs.get("repetition_decay"), fallback_decay, 0
         )
 
+        # DRY options
+        dry_multiplier = unwrap(kwargs.get("dry_multiplier"), 0.0)
+
+        # < 0 = disabled
+        if dry_multiplier > 0:
+            gen_settings.dry_multiplier = dry_multiplier
+
+            # TODO: Maybe set the "sane" defaults instead?
+            gen_settings.dry_allowed_length = unwrap(
+                kwargs.get("dry_allowed_length"), 0
+            )
+            gen_settings.dry_base = unwrap(kwargs.get("dry_base"), 0.0)
+
+            # Exl2 has dry_range as 0 for unlimited unlike -1 for penalty_range
+            # Use max_seq_len as the fallback to stay consistent
+            gen_settings.dry_range = unwrap(
+                kwargs.get("dry_range"), self.config.max_seq_len
+            )
+
+            # Tokenize sequence breakers
+            dry_sequence_breakers_json = kwargs.get("dry_sequence_breakers")
+            if dry_sequence_breakers_json:
+                gen_settings.dry_sequence_breakers = {
+                    self.encode_tokens(s)[-1] for s in dry_sequence_breakers_json
+                }
+
         # Initialize grammar handler
         grammar_handler = ExLlamaV2Grammar()
 
@@ -1130,7 +1177,8 @@ async def generate_gen(
             )
 
         # Store the gen settings for logging purposes
-        gen_settings_log_dict = vars(gen_settings)
+        # Deepcopy to save a snapshot of vars
+        gen_settings_log_dict = deepcopy(vars(gen_settings))
 
         # Set banned tokens
         banned_tokens = unwrap(kwargs.get("banned_tokens"), [])

Diff for: common/sampling.py

@@ -1,5 +1,6 @@
 """Common functions for sampling parameters"""
 
+import json
 import pathlib
 import yaml
 from copy import deepcopy
@@ -140,6 +141,28 @@ class BaseSamplerRequest(BaseModel):
         default_factory=lambda: get_default_sampler_value("repetition_decay", 0)
     )
 
+    dry_multiplier: Optional[float] = Field(
+        default_factory=lambda: get_default_sampler_value("dry_multiplier", 0.0)
+    )
+
+    dry_base: Optional[float] = Field(
+        default_factory=lambda: get_default_sampler_value("dry_base", 0.0)
+    )
+
+    dry_allowed_length: Optional[int] = Field(
+        default_factory=lambda: get_default_sampler_value("dry_allowed_length", 0)
+    )
+
+    dry_range: Optional[int] = Field(
+        default_factory=lambda: get_default_sampler_value("dry_range", 0),
+        alias=AliasChoices("dry_range", "dry_penalty_last_n"),
+        description=("Aliases: dry_penalty_last_n"),
+    )
+
+    dry_sequence_breakers: Optional[Union[str, List[str]]] = Field(
+        default_factory=lambda: get_default_sampler_value("dry_sequence_breakers", [])
+    )
+
     mirostat_mode: Optional[int] = Field(
         default_factory=lambda: get_default_sampler_value("mirostat_mode", 0)
     )
@@ -305,6 +328,17 @@ def to_gen_params(self, **kwargs):
                 int(x) for x in self.allowed_tokens.split(",") if x.isdigit()
             ]
 
+        # Convert sequence breakers into an array of strings
+        # NOTE: This sampler sucks to parse.
+        if self.dry_sequence_breakers and isinstance(self.dry_sequence_breakers, str):
+            if not self.dry_sequence_breakers.startswith("["):
+                self.dry_sequence_breakers = f"[{self.dry_sequence_breakers}]"
+
+            try:
+                self.dry_sequence_breakers = json.loads(self.dry_sequence_breakers)
+            except Exception:
+                self.dry_sequence_breakers = []
+
         gen_params = {
             "max_tokens": self.max_tokens,
             "min_tokens": self.min_tokens,
@@ -335,6 +369,11 @@ def to_gen_params(self, **kwargs):
             "presence_penalty": self.presence_penalty,
             "repetition_penalty": self.repetition_penalty,
             "penalty_range": self.penalty_range,
+            "dry_multiplier": self.dry_multiplier,
+            "dry_base": self.dry_base,
+            "dry_allowed_length": self.dry_allowed_length,
+            "dry_sequence_breakers": self.dry_sequence_breakers,
+            "dry_range": self.dry_range,
             "repetition_decay": self.repetition_decay,
             "mirostat": self.mirostat_mode == 2,
             "mirostat_tau": self.mirostat_tau,

Diff for: common/templating.py

@@ -5,6 +5,7 @@
 from importlib.metadata import version as package_version
 from typing import List, Optional
 from jinja2 import Template, TemplateError
+from jinja2.ext import loopcontrols
 from jinja2.sandbox import ImmutableSandboxedEnvironment
 from loguru import logger
 from packaging import version
@@ -32,7 +33,10 @@ class PromptTemplate:
     raw_template: str
     template: Template
     environment: ImmutableSandboxedEnvironment = ImmutableSandboxedEnvironment(
-        trim_blocks=True, lstrip_blocks=True, enable_async=True
+        trim_blocks=True,
+        lstrip_blocks=True,
+        enable_async=True,
+        extensions=[loopcontrols],
     )
     metadata: Optional[TemplateMetadata] = None
 
@@ -106,20 +110,26 @@ def __init__(self, name: str, raw_template: str):
         self.template = self.compile(raw_template)
 
     @classmethod
-    def from_file(self, prompt_template_name: str):
+    def from_file(self, template_path: pathlib.Path):
         """Get a template from a jinja file."""
 
-        template_path = pathlib.Path(f"templates/{prompt_template_name}.jinja")
+        # Add the jinja extension if it isn't provided
+        if template_path.suffix.endswith(".jinja"):
+            template_name = template_path.name.split(".jinja")[0]
+        else:
+            template_name = template_path.name
+            template_path = template_path.with_suffix(".jinja")
+
         if template_path.exists():
             with open(template_path, "r", encoding="utf8") as raw_template_stream:
                 return PromptTemplate(
-                    name=prompt_template_name,
+                    name=template_name,
                     raw_template=raw_template_stream.read(),
                 )
         else:
             # Let the user know if the template file isn't found
             raise TemplateLoadError(
-                f'Chat template "{prompt_template_name}" not found in files.'
+                f'Chat template "{template_name}" not found in files.'
             )
 
     @classmethod

Diff for: endpoints/Kobold/router.py

@@ -137,7 +137,7 @@ async def get_version():
 async def get_extra_version():
     """Impersonate Koboldcpp."""
 
-    return {"result": "KoboldCpp", "version": "1.61"}
+    return {"result": "KoboldCpp", "version": "1.71"}
 
 
 @kai_router.get("/config/soft_prompts_list")

Diff for: endpoints/core/router.py

@@ -103,7 +103,7 @@ async def list_draft_models(request: Request) -> ModelList:
 
         models = get_model_list(draft_model_path.resolve())
     else:
-        models = await get_current_model_list(is_draft=True)
+        models = await get_current_model_list(model_type="draft")
 
     return models
 

Diff for: sampler_overrides/sample_preset.yml

@@ -97,6 +97,24 @@ penalty_range:
   override: -1
   force: false
 
+# MARK: DRY
+dry_multiplier:
+  override: 0.0
+  force: false
+dry_base:
+  override: 0.0
+  force: false
+dry_allowed_length:
+  override: 0
+  force: false
+dry_range:
+  override: 0
+  force: false
+dry_sequence_breakers:
+  override: []
+  force: false
+  additive: false
+
 # MARK: Mirostat
 mirostat_mode:
   override: 0

Diff for: templates/alpaca.jinja

@@ -1,5 +1,5 @@
 {# Metadata #}
-{% set stop_strings = ["### Instruction:", "### Input:", "### Response:"] %}
+{%- set stop_strings = ["### Instruction:", "### Input:", "### Response:"] -%}
 
 {# Template #}
 {{ (messages|selectattr('role', 'equalto', 'system')|list|last).content|trim if (messages|selectattr('role', 'equalto', 'system')|list) else '' }}

Diff for: templates/chatml.jinja

@@ -1,5 +1,5 @@
 {# Metadata #} 
-{% set stop_strings = ["<|im_start|>", "<|im_end|>"] %}
+{%- set stop_strings = ["<|im_start|>", "<|im_end|>"] -%}
 
 {# Template #}
 {% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content']}}{% if (loop.last and add_generation_prompt) or not loop.last %}{{ '<|im_end|>' + '\n'}}{% endif %}{% endfor %}

Diff for: templates/chatml_with_headers_tool_calling.jinja renamed to templates/tool_calls/chatml_with_headers.jinja

@@ -1,8 +1,8 @@
 {# Metadata #} 
-{% set stop_strings = ["<|im_start|>", "<|im_end|>"] %}
-{% set message_roles = ['system', 'user', 'assistant', 'tool'] %}
-{% set tool_start = "<|tool_start|>" %}
-{% set tool_end = "<|tool_end|>" %}
+{%- set stop_strings = ["<|im_start|>", "<|im_end|>"] -%}
+{%- set message_roles = ['system', 'user', 'assistant', 'tool'] -%}
+{%- set tool_start = "<|tool_start|>" -%}
+{%- set tool_end = "<|tool_end|>" -%}
 {%- set start_header = "<|start_header_id|>" -%}
 {%- set end_header = "<|end_header_id|>\n" -%}