Skip to content

Commit 1355b2c

Browse files
ATMxsp01ATMxsp01
ATMxsp01
and
ATMxsp01
authored
Add model Qwen-VL-Chat to iGPU-perf (#11558)
* Add model Qwen-VL-Chat to iGPU-perf * small fix --------- Co-authored-by: ATMxsp01 <[email protected]>
1 parent 105e124 commit 1355b2c

File tree

8 files changed

+40
-3
lines changed

8 files changed

+40
-3
lines changed

‎.github/workflows/llm_performance_tests.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -554,7 +554,7 @@ jobs:
554554
pip install --upgrade pip
555555
pip install --upgrade wheel
556556
pip install --upgrade omegaconf pandas
557-
pip install --upgrade tiktoken einops transformers_stream_generator
557+
pip install --upgrade tiktoken einops transformers_stream_generator matplotlib
558558
559559
cd python\llm
560560
python setup.py clean --all bdist_wheel --win
@@ -584,7 +584,7 @@ jobs:
584584
pip install --upgrade pip
585585
pip install --upgrade wheel
586586
pip install --upgrade omegaconf pandas
587-
pip install --upgrade tiktoken einops transformers_stream_generator
587+
pip install --upgrade tiktoken einops transformers_stream_generator matplotlib
588588
589589
pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
590590
pip show ipex-llm | findstr %TEST_VERSION_DATE%

‎python/llm/dev/benchmark/all-in-one/run.py

+27
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@
4444

4545
PHI3VISION_IDS = ['microsoft/phi-3-vision-128k-instruct']
4646

47+
QWENVL_IDS = ['Qwen/Qwen-VL-Chat']
48+
4749
results = []
4850
excludes = []
4951

@@ -923,6 +925,12 @@ def run_transformer_int4_gpu_win(repo_id,
923925
trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding).eval()
924926
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
925927
model = model.to('xpu')
928+
elif repo_id in QWENVL_IDS:
929+
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
930+
modules_to_not_convert=['c_fc', 'out_proj'],
931+
trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding).eval()
932+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
933+
model = model.to('xpu')
926934
else:
927935
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
928936
trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding).eval()
@@ -1038,6 +1046,13 @@ def run_transformer_int4_fp16_gpu_win(repo_id,
10381046
torch_dtype=torch.float16).eval()
10391047
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
10401048
model = model.to('xpu')
1049+
elif repo_id in QWENVL_IDS:
1050+
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
1051+
modules_to_not_convert=['c_fc', 'out_proj'],
1052+
trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding,
1053+
torch_dtype=torch.float16).eval()
1054+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
1055+
model = model.to('xpu')
10411056
else:
10421057
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
10431058
trust_remote_code=True, use_cache=True, cpu_embedding=cpu_embedding,
@@ -1149,6 +1164,12 @@ def run_transformer_int4_loadlowbit_gpu_win(repo_id,
11491164
use_cache=True, cpu_embedding=cpu_embedding).eval()
11501165
tokenizer = AutoTokenizer.from_pretrained(model_path+'-'+low_bit, trust_remote_code=True)
11511166
model = model.to('xpu')
1167+
elif repo_id in QWENVL_IDS:
1168+
model = AutoModelForCausalLM.load_low_bit(model_path+'-'+low_bit, optimize_model=True, trust_remote_code=True,
1169+
modules_to_not_convert=['c_fc', 'out_proj'],
1170+
use_cache=True, cpu_embedding=cpu_embedding).eval()
1171+
tokenizer = AutoTokenizer.from_pretrained(model_path+'-'+low_bit, trust_remote_code=True)
1172+
model = model.to('xpu')
11521173
else:
11531174
model = AutoModelForCausalLM.load_low_bit(model_path+'-'+low_bit, optimize_model=True, trust_remote_code=True,
11541175
use_cache=True, cpu_embedding=cpu_embedding).eval()
@@ -1259,6 +1280,12 @@ def run_transformer_int4_fp16_loadlowbit_gpu_win(repo_id,
12591280
use_cache=True, cpu_embedding=cpu_embedding).eval()
12601281
tokenizer = AutoTokenizer.from_pretrained(model_path+'-'+low_bit, trust_remote_code=True)
12611282
model = model.half().to('xpu')
1283+
elif repo_id in QWENVL_IDS:
1284+
model = AutoModelForCausalLM.load_low_bit(model_path+'-'+low_bit, optimize_model=True, trust_remote_code=True,
1285+
modules_to_not_convert=['c_fc', 'out_proj'],
1286+
use_cache=True, cpu_embedding=cpu_embedding).eval()
1287+
tokenizer = AutoTokenizer.from_pretrained(model_path+'-'+low_bit, trust_remote_code=True)
1288+
model = model.half().to('xpu')
12621289
else:
12631290
model = AutoModelForCausalLM.load_low_bit(model_path+'-'+low_bit, optimize_model=True, trust_remote_code=True,
12641291
use_cache=True, cpu_embedding=cpu_embedding).eval()

‎python/llm/dev/benchmark/all-in-one/save.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import sys
2424
import gc
2525

26-
from run import LLAMA_IDS, CHATGLM_IDS, LLAVA_IDS, PHI3VISION_IDS, get_model_path
26+
from run import LLAMA_IDS, CHATGLM_IDS, LLAVA_IDS, PHI3VISION_IDS, QWENVL_IDS, get_model_path
2727

2828
current_dir = os.path.dirname(os.path.realpath(__file__))
2929

@@ -57,6 +57,11 @@ def save_model_in_low_bit(repo_id,
5757
modules_to_not_convert=["vision_embed_tokens"],
5858
trust_remote_code=True, use_cache=True).eval()
5959
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
60+
elif repo_id in QWENVL_IDS:
61+
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
62+
modules_to_not_convert=['c_fc', 'out_proj'],
63+
trust_remote_code=True, use_cache=True).eval()
64+
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
6065
else:
6166
model = AutoModelForCausalLM.from_pretrained(model_path, optimize_model=True, load_in_low_bit=low_bit,
6267
trust_remote_code=True, use_cache=True).eval()

‎python/llm/test/benchmark/igpu-perf/1024-128.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ repo_id:
1212
- 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
1313
- 'RWKV/v5-Eagle-7B-HF'
1414
- '01-ai/Yi-6B-Chat'
15+
- 'Qwen/Qwen-VL-Chat'
1516
local_model_hub: 'path to your local model hub'
1617
warm_up: 1
1718
num_trials: 3

‎python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ repo_id:
1111
- 'mistralai/Mistral-7B-Instruct-v0.2'
1212
- 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
1313
- '01-ai/Yi-6B-Chat'
14+
- 'Qwen/Qwen-VL-Chat'
1415
local_model_hub: 'path to your local model hub'
1516
warm_up: 1
1617
num_trials: 3

‎python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_loadlowbit.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ repo_id:
1111
- 'mistralai/Mistral-7B-Instruct-v0.2'
1212
- 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
1313
- '01-ai/Yi-6B-Chat'
14+
- 'Qwen/Qwen-VL-Chat'
1415
local_model_hub: 'path to your local model hub'
1516
warm_up: 1
1617
num_trials: 3

‎python/llm/test/benchmark/igpu-perf/2048-256_int4_fp16.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ repo_id:
1111
- 'mistralai/Mistral-7B-Instruct-v0.2'
1212
- 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
1313
- '01-ai/Yi-6B-Chat'
14+
- 'Qwen/Qwen-VL-Chat'
1415
local_model_hub: 'path to your local model hub'
1516
warm_up: 1
1617
num_trials: 3

‎python/llm/test/benchmark/igpu-perf/32-32_int4_fp16.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ repo_id:
1111
- 'mistralai/Mistral-7B-Instruct-v0.2'
1212
- 'deepseek-ai/deepseek-coder-7b-instruct-v1.5'
1313
- '01-ai/Yi-6B-Chat'
14+
- 'Qwen/Qwen-VL-Chat'
1415
local_model_hub: 'path to your local model hub'
1516
warm_up: 3
1617
num_trials: 5

0 commit comments

Comments
 (0)