Skip to content

Commit 9cfe071

Browse files
yeqcharlottewukaixingxp
authored andcommitted
[Frontend] Added chat templates for LLaMa4 pythonic tool calling (vllm-project#16463)
Signed-off-by: Ye (Charlotte) Qi <[email protected]> Co-authored-by: Kai Wu <[email protected]>
1 parent b7643a9 commit 9cfe071

File tree

5 files changed

+182
-2
lines changed

5 files changed

+182
-2
lines changed

docs/source/features/tool_calling.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,8 @@ Example supported models:
245245
* `meta-llama/Llama-3.2-3B-Instruct`\* (use with `examples/tool_chat_template_llama3.2_pythonic.jinja`)
246246
* `Team-ACE/ToolACE-8B` (use with `examples/tool_chat_template_toolace.jinja`)
247247
* `fixie-ai/ultravox-v0_4-ToolACE-8B` (use with `examples/tool_chat_template_toolace.jinja`)
248+
* `meta-llama/Llama-4-Scout-17B-16E-Instruct`\* (use with `examples/tool_chat_template_llama4_pythonic.jinja`)
249+
* `meta-llama/Llama-4-Maverick-17B-128E-Instruct`\* (use with `examples/tool_chat_template_llama4_pythonic.jinja`)
248250

249251
Flags: `--tool-call-parser pythonic --chat-template {see_above}`
250252

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
{{- bos_token }}
2+
{%- if custom_tools is defined %}
3+
{%- set tools = custom_tools %}
4+
{%- endif %}
5+
{%- if not tools_in_user_message is defined %}
6+
{%- set tools_in_user_message = false %}
7+
{%- endif %}
8+
{%- if not tools is defined %}
9+
{%- set tools = none %}
10+
{%- endif %}
11+
12+
{#- This block extracts the system message, so we can slot it into the right place. #}
13+
{%- if messages[0]['role'] == 'system' %}
14+
{%- if messages[0]['content'] is string %}
15+
{%- set system_message = messages[0]['content']|trim %}
16+
{%- else %}
17+
{%- set system_message = messages[0]['content'][0]['text']|trim %}
18+
{%- endif %}
19+
{%- set messages = messages[1:] %}
20+
{%- else %}
21+
{%- if tools is not none %}
22+
{#- Add default tool system message when tools are provided #}
23+
{%- set system_message = "You are a helpful assistant with tool calling "
24+
"capabilities. Only reply with a tool call if the function exists in the "
25+
"library provided by the user. If it doesn't exist, just reply directly in "
26+
"natural language. When you receive a tool call response, use the output to "
27+
"format an answer to the original user question." %}
28+
{%- else %}
29+
{%- set system_message = "" %}
30+
{%- endif %}
31+
{%- endif %}
32+
33+
{#- System message if the user supplied one, or if tools are used (default tool system message) #}
34+
{%- if system_message %}
35+
{#- always use user provided system message to override default tool system message #}
36+
{{- "<|header_start|>system<|header_end|>\n\n" }}
37+
{{- system_message }}
38+
{%- if tools is not none and not tools_in_user_message %}
39+
{{- "Tools: You have access to the following tools. You might need to use one "
40+
"or more function/tool calls to fulfill the task. \n"
41+
"If none are needed, then proceed to the response.\n\n"
42+
"Tool Call Syntax: You can call tools using the following syntax:\n"
43+
"[func_name1(params_name1=params_value1, params_name2=params_value2, ...), ...]\n"
44+
"Do not include anything else when calling the tools with the syntax above.\n\n"
45+
"Here is a list of functions in JSON format that you can invoke.\n " }}
46+
{%- for t in tools %}
47+
{{- t | tojson(indent=4) }}
48+
{{- "\n\n" }}
49+
{%- endfor %}
50+
{%- endif %}
51+
{{- "<|eot|>" }}
52+
{%- endif %}
53+
54+
{#- Custom tools are passed in a user message with some extra guidance #}
55+
{%- if tools_in_user_message and tools is not none %}
56+
{#- Extract the first user message so we can plug it in here #}
57+
{%- if messages | length != 0 %}
58+
{%- if messages[0]['content'] is string %}
59+
{%- set first_user_message = messages[0]['content']|trim %}
60+
{%- else %}
61+
{%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
62+
{%- endif %}
63+
{%- set messages = messages[1:] %}
64+
{%- else %}
65+
{{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
66+
{%- endif %}
67+
{{- '<|header_start|>user<|header_end|>\n\n' -}}
68+
{{- first_user_message}}
69+
{{- "\nHere is a list of functions in JSON format that you can invoke:"}}
70+
{%- for t in tools %}
71+
{{- t | tojson(indent=4) }}
72+
{{- "\n\n" }}
73+
{%- endfor %}
74+
{{- "Should you decide to return the function call(s), put them in the format "
75+
"of [func_name1(params_name1=params_value1, params_name2=params_value2, "
76+
"...), ...]\nDo not include anything else when calling the tools with the "
77+
"syntax above." }}
78+
{%- endif %}
79+
80+
{%- for message in messages %}
81+
{%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
82+
{{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
83+
{%- if message['content'] is string %}
84+
{{- message['content'] }}
85+
{%- else %}
86+
{%- for content in message['content'] %}
87+
{%- if content['type'] == 'image' %}
88+
{{- '<|image|>' }}
89+
{%- elif content['type'] == 'text' %}
90+
{{- content['text'] | trim }}
91+
{%- endif %}
92+
{%- endfor %}
93+
{%- endif %}
94+
{{- "<|eot|>" }}
95+
{%- elif 'tool_calls' in message and message.tool_calls|length > 0 %}
96+
{%- set tool_call = message.tool_calls[0].function %}
97+
{{- '<|header_start|>assistant<|header_end|>\n\n' -}}
98+
{%- if message['content'] is string %}
99+
{{- message['content'] }}
100+
{%- else %}
101+
{%- for content in message['content'] %}
102+
{%- if content['type'] == 'image' %}
103+
{{- '<|image|>' }}
104+
{%- elif content['type'] == 'text' %}
105+
{{- content['text'] }}
106+
{%- endif %}
107+
{%- endfor %}
108+
{%- endif %}
109+
{%- for tool_call in message.tool_calls %}
110+
{%- if tool_call.function is defined %}
111+
{%- set tool_call = tool_call.function %}
112+
{%- endif %}
113+
{{- tool_call.name + '(' -}}
114+
{%- for param in tool_call.arguments %}
115+
{{- param + '=' -}}
116+
{{- "%s" | format(tool_call.arguments[param]) -}}
117+
{% if not loop.last %}, {% endif %}
118+
{%- endfor %}
119+
{{- ')' -}}
120+
{% if not loop.last %}, {% endif %}
121+
{%- endfor %}
122+
{{- "<|eom|>" }}
123+
{%- elif message.role == "tool" or message.role == "ipython" %}
124+
{{- "<|header_start|>ipython<|header_end|>\n\n" }}
125+
{%- if message.content is string %}
126+
{{- message.content | tojson }}
127+
{%- else %}
128+
{%- for content in message['content'] %}
129+
{%- if content['type'] == 'text' %}
130+
{{- content['text'] | tojson }}
131+
{%- endif %}
132+
{%- endfor %}
133+
{%- endif %}
134+
{{- "<|eom|>" }}
135+
{%- endif %}
136+
{%- endfor %}
137+
{%- if add_generation_prompt %}
138+
{{- '<|header_start|>assistant<|header_end|>\n\n' }}
139+
{%- endif %}

tests/tool_use/conftest.py

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,33 @@
1010
from .utils import ARGS, CONFIGS, ServerConfig
1111

1212

13+
# select models to test based on command line arguments
14+
def pytest_addoption(parser):
15+
parser.addoption("--models",
16+
nargs="+",
17+
help="Specify one or more models to test")
18+
parser.addoption("--extended",
19+
action="store_true",
20+
default=False,
21+
help="invoke extended tests requiring large GPUs")
22+
23+
1324
# for each server config, download the model and return the config
1425
@pytest.fixture(scope="session", params=CONFIGS.keys())
1526
def server_config(request):
16-
config = CONFIGS[request.param]
27+
extended = request.config.getoption("--extended")
28+
models = request.config.getoption("--models")
29+
30+
config_keys_to_test = [
31+
key for key in CONFIGS if (models is None or key in models) and (
32+
extended or not CONFIGS[key].get("extended", False))
33+
]
34+
35+
config_key = request.param
36+
if config_key not in config_keys_to_test:
37+
pytest.skip(f"Skipping config '{config_key}'")
38+
39+
config = CONFIGS[config_key]
1740

1841
if current_platform.is_rocm() and not config.get("supports_rocm", True):
1942
pytest.skip("The {} model can't be tested on the ROCm platform".format(

tests/tool_use/utils.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ class ServerConfig(TypedDict, total=False):
1616
system_prompt: Optional[str]
1717
supports_parallel: Optional[bool]
1818
supports_rocm: Optional[bool]
19+
extended: Optional[bool] # tests do not run in CI automatically
1920

2021

2122
def patch_system_prompt(messages: list[dict[str, Any]],
@@ -82,6 +83,21 @@ def ensure_system_prompt(messages: list[dict[str, Any]],
8283
"supports_parallel":
8384
False,
8485
},
86+
"llama4": {
87+
"model":
88+
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
89+
"arguments": [
90+
"--enforce-eager", "--no-enable-prefix-caching",
91+
"--tool-call-parser", "pythonic", "--chat-template",
92+
str(VLLM_PATH /
93+
"examples/tool_chat_template_llama4_pythonic.jinja"), "-tp",
94+
"4"
95+
],
96+
"supports_parallel":
97+
False,
98+
"extended":
99+
True
100+
},
85101
"mistral": {
86102
"model":
87103
"mistralai/Mistral-7B-Instruct-v0.3",

vllm/entrypoints/openai/tool_parsers/pythonic_tool_parser.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ class _UnexpectedAstError(Exception):
2828
class PythonicToolParser(ToolParser):
2929
"""
3030
Tool call parser for models that produce tool calls in a pythonic style,
31-
such as Llama 3.2 models.
31+
such as Llama 3.2 and Llama 4 models.
3232
3333
Used when --enable-auto-tool-choice --tool-call-parser pythonic are all set
3434
"""

0 commit comments

Comments
 (0)