Skip to content

Commit bb83fd5

Browse files
maxdebayserMu Huai
authored and
Mu Huai
committed
Add chat template for Llama 4 models (vllm-project#16428)
Signed-off-by: Max de Bayser <[email protected]> Signed-off-by: Mu Huai <[email protected]>
1 parent 9700a6a commit bb83fd5

File tree

4 files changed

+139
-1
lines changed

4 files changed

+139
-1
lines changed

docs/source/features/tool_calling.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,10 +152,11 @@ Recommended flags: `--tool-call-parser mistral --chat-template examples/tool_cha
152152

153153
Supported models:
154154

155-
All Llama 3.1 and 3.2 models should be supported.
155+
All Llama 3.1, 3.2 and 4 models should be supported.
156156

157157
* `meta-llama/Llama-3.1-*`
158158
* `meta-llama/Llama-3.2-*`
159+
* `meta-llama/Llama-4-*`
159160

160161
The tool calling that is supported is the [JSON based tool calling](https://llama.meta.com/docs/model-cards-and-prompt-formats/llama3_1/#json-based-tool-calling). For [pythonic tool calling](https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#zero-shot-function-calling) introduced by the Llama-3.2 models, see the `pythonic` tool parser below.
161162

@@ -176,6 +177,12 @@ images.
176177

177178
Recommended flags: `--tool-call-parser llama3_json --chat-template {see_above}`
178179

180+
VLLM also provides a JSON based chat template for Llama 4:
181+
* `examples/tool_chat_template_llama4_json.jinja` - this is based on the "official" chat template for the Llama 4
182+
models, but tweaked so that it works better with vLLM.
183+
184+
For Llama 4 use `--tool-call-parser llama4_json examples/tool_chat_template_llama4_json.jinja`.
185+
179186
#### IBM Granite
180187

181188
Supported models:
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
{%- macro is_array_of_type_objects(var) -%}
2+
{%- if var is iterable and var is not string -%}
3+
{%- set valid = true -%}
4+
{%- for item in var -%}
5+
{%- if 'type' not in item -%}
6+
{%- set valid = false -%}
7+
{%- break -%}
8+
{%- endif -%}
9+
{%- endfor -%}
10+
{{ valid }}
11+
{%- else -%}
12+
{{ false }}
13+
{%- endif -%}
14+
{%- endmacro %}
15+
16+
{%- macro render_message(message) %}
17+
{%- if message['content'] is string %}
18+
{{- message['content']|trim }}
19+
{%- elif is_array_of_type_objects(data) == 'True' %}
20+
{%- for content in message['content'] %}
21+
{%- if content['type'] == 'image' %}
22+
{{- '<|image|>' }}
23+
{%- elif content['type'] == 'text' %}
24+
{{- content['text']|trim }}
25+
{%- endif %}
26+
{%- endfor %}
27+
{%- else %}
28+
{{- message['content']|tojson }}
29+
{%- endif %}
30+
{%- endmacro %}
31+
32+
{{- bos_token }}
33+
{%- if custom_tools is defined %}
34+
{%- set tools = custom_tools %}
35+
{%- endif %}
36+
{%- if not tools_in_user_message is defined %}
37+
{%- set tools_in_user_message = true %}
38+
{%- endif %}
39+
{%- if not tools is defined %}
40+
{%- set tools = none %}
41+
{%- endif %}
42+
43+
{#- This block extracts the system message, so we can slot it into the right place. #}
44+
{%- if messages[0]['role'] == 'system' %}
45+
{%- set system_message = messages[0] %}
46+
{%- set messages = messages[1:] %}
47+
{%- else %}
48+
{%- set system_message = ({ "content": "You are a helpful assistant with tool calling "
49+
"capabilities. Only reply with a tool call if the function exists in the "
50+
"library provided by the user. If it doesn't exist, just reply directly in "
51+
"natural language. When you receive a tool call response, use the output to "
52+
"format an answer to the original user question."}) %}
53+
{%- endif %}
54+
55+
{%- set tool_lib_preamble = 'Tools: You have access to the following tools. You might need to use one '
56+
'or more function/tool calls to fulfill the task. \n'
57+
'If none are needed, then proceed to the response.\n\n'
58+
'Tool Call Syntax: You can call tools using the following syntax:\n'
59+
'{"name": function name, "parameters": dictionary of argument name and its value}.\n'
60+
'Separate multiple function calls by "; ". Do not use variables.\n'
61+
'Do not include anything else when calling the tools with the syntax above.\n\n'
62+
'Here is a list of functions in JSON format that you can invoke.\n' %}
63+
64+
{{- "<|header_start|>system<|header_end|>\n\n" }}
65+
{%- if tools is not none and not tools_in_user_message %}
66+
{{- tool_lib_preamble }}
67+
{%- for t in tools %}
68+
{{- t | tojson(indent=4) }}
69+
{{- "\n\n" }}
70+
{%- endfor %}
71+
{%- endif %}
72+
{{- render_message(system_message) }}
73+
{{ "<|eot|>\n" }}
74+
75+
{#- Custom tools are passed in a user message with some extra guidance #}
76+
{%- if tools_in_user_message and not tools is none %}
77+
{#- Extract the first user message so we can plug it in here #}
78+
{%- if messages | length != 0 %}
79+
{%- set first_user_message = messages[0] %}
80+
{%- set messages = messages[1:] %}
81+
{%- else %}
82+
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
83+
{%- endif %}
84+
{{- '<|header_start|>user<|header_end|>\n\n' }}
85+
{{- tool_lib_preamble }}
86+
{%- for t in tools %}
87+
{{- t | tojson(indent=4) }}
88+
{{- "\n\n" }}
89+
{%- endfor %}
90+
{{- render_message(first_user_message) + "\n<|eot|>"}}
91+
{%- endif %}
92+
93+
{%- for message in messages %}
94+
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
95+
{{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
96+
{{- render_message(message) }}
97+
{{- "\n<|eot|>" }}
98+
{%- elif 'tool_calls' in message and message.tool_calls|length > 0 %}
99+
{{- '\n<|header_start|>assistant<|header_end|>\n\n' -}}
100+
{{- render_message(message) }}
101+
{%- for tool_call in message.tool_calls %}
102+
{{- '{"name": "' + tool_call.function.name + '", ' }}
103+
{{- '"parameters": ' }}
104+
{{- tool_call.function.arguments | tojson }}
105+
{{- "}" }}
106+
{%- endfor %}
107+
{{- "\n<|eot|>" }}
108+
{%- elif message.role == "tool" or message.role == "ipython" %}
109+
{{- "\n<|header_start|>ipython<|header_end|>\n\n" }}
110+
{{- render_message(message) }}
111+
{{- "\n<|eom|>" }}
112+
{%- endif %}
113+
{%- endfor %}
114+
{%- if add_generation_prompt %}
115+
{{- '\n<|header_start|>assistant<|header_end|>\n\n' }}
116+
{%- endif %}

tests/tool_use/utils.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,20 @@ def ensure_system_prompt(messages: list[dict[str, Any]],
9898
"extended":
9999
True
100100
},
101+
"llama4_json": {
102+
"model":
103+
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
104+
"arguments": [
105+
"--enforce-eager", "--no-enable-prefix-caching", "-tp", "4",
106+
"--distributed-executor-backend", "mp", "--tool-call-parser",
107+
"llama4_json", "--chat-template",
108+
str(VLLM_PATH / "examples/tool_chat_template_llama4_json.jinja")
109+
],
110+
"supports_parallel":
111+
True,
112+
"extended":
113+
True
114+
},
101115
"mistral": {
102116
"model":
103117
"mistralai/Mistral-7B-Instruct-v0.3",

vllm/entrypoints/openai/tool_parsers/llama_tool_parser.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727

2828

2929
@ToolParserManager.register_module("llama3_json")
30+
@ToolParserManager.register_module("llama4_json")
3031
class Llama3JsonToolParser(ToolParser):
3132
"""
3233
Tool call parser for Llama 3.1 models intended for use with the

0 commit comments

Comments
 (0)