File tree 1 file changed +2
-4
lines changed 1 file changed +2
-4
lines changed Original file line number Diff line number Diff line change @@ -146,10 +146,9 @@ python3 vllm/benchmarks/benchmark_serving.py \
146
146
147
147
``` bash
148
148
VLLM_USE_V1=1 vllm serve meta-llama/Meta-Llama-3-8B-Instruct \
149
- --speculative-model " [ngram]" \
150
149
--ngram_prompt_lookup_min 2 \
151
150
--ngram-prompt-lookup-max 5 \
152
- --num_speculative_tokens 5
151
+ --speculative_config ' {"model": "[ngram]", " num_speculative_tokens": 5}
153
152
```
154
153
155
154
``` bash
@@ -274,10 +273,9 @@ python3 vllm/benchmarks/benchmark_throughput.py \
274
273
--output-len=100 \
275
274
--num-prompts=2048 \
276
275
--async-engine \
277
- --speculative-model=" [ngram]" \
278
276
--ngram_prompt_lookup_min=2 \
279
277
--ngram-prompt-lookup-max=5 \
280
- --num_speculative_tokens=5
278
+ --speculative_config ' { " model " : " [ngram] " , " num_speculative_tokens" : 5}
281
279
```
282
280
283
281
```
You can’t perform that action at this time.
0 commit comments