File tree 1 file changed +3
-6
lines changed
1 file changed +3
-6
lines changed Original file line number Diff line number Diff line change 20
20
Learn more about LMCache environment setup, please refer to:
21
21
https://docs.lmcache.ai/getting_started/installation.html
22
22
"""
23
+ import argparse
23
24
import contextlib
24
25
import os
25
26
import time
26
- import argparse
27
27
28
28
from lmcache .experimental .cache_engine import LMCacheEngineBuilder
29
29
from lmcache .integration .vllm .utils import ENGINE_NAME
@@ -45,11 +45,8 @@ def setup_environment_variables():
45
45
46
46
47
47
@contextlib .contextmanager
48
- def build_llm_with_lmcache (
49
- lmcache_connector : str ,
50
- model : str ,
51
- vllm_version : str
52
- ):
48
+ def build_llm_with_lmcache (lmcache_connector : str , model : str ,
49
+ vllm_version : str ):
53
50
ktc = KVTransferConfig .from_cli (
54
51
f'{{"kv_connector":"{ lmcache_connector } ", "kv_role":"kv_both"}}' )
55
52
# Set GPU memory utilization to 0.8 for an A40 GPU with 40GB
You can’t perform that action at this time.
0 commit comments