Skip to content

Commit 5ecdb10

Browse files
ggerganovggerganov
committed
batched-bench : add readme + n_kv_max is now configurable
1 parent 7438728 commit 5ecdb10

File tree

4 files changed

+73
-12
lines changed

4 files changed

+73
-12
lines changed

Diff for: ‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ models-mnt
5555
/server
5656
/simple
5757
/batched
58+
/batched-bench
5859
/export-lora
5960
/finetune
6061
/speculative

Diff for: ‎Makefile

+11-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
# Define the default target now so that it is always the first target
2-
BUILD_TARGETS = main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml simple batched save-load-state server embd-input-test gguf llama-bench baby-llama beam-search speculative infill benchmark-matmult parallel finetune export-lora tests/test-c.o
2+
BUILD_TARGETS = \
3+
main quantize quantize-stats perplexity embedding vdot q8dot train-text-from-scratch convert-llama2c-to-ggml \
4+
simple batched batched-bench save-load-state server embd-input-test gguf llama-bench baby-llama beam-search \
5+
speculative infill benchmark-matmult parallel finetune export-lora tests/test-c.o
36

47
# Binaries only useful for tests
5-
TEST_TARGETS = tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe
8+
TEST_TARGETS = \
9+
tests/test-llama-grammar tests/test-grammar-parser tests/test-double-float tests/test-grad0 tests/test-opt \
10+
tests/test-quantize-fns tests/test-quantize-perf tests/test-sampling tests/test-tokenizer-0-llama \
11+
tests/test-tokenizer-0-falcon tests/test-tokenizer-1-llama tests/test-tokenizer-1-bpe
612

713
# Code coverage output files
814
COV_TARGETS = *.gcno tests/*.gcno *.gcda tests/*.gcda *.gcov tests/*.gcov lcov-report gcovr-report
@@ -554,6 +560,9 @@ simple: examples/simple/simple.cpp build-info.h ggml.
554560
batched: examples/batched/batched.cpp build-info.h ggml.o llama.o common.o $(OBJS)
555561
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
556562

563+
batched-bench: examples/batched-bench/batched-bench.cpp build-info.h ggml.o llama.o common.o $(OBJS)
564+
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
565+
557566
quantize: examples/quantize/quantize.cpp build-info.h ggml.o llama.o $(OBJS)
558567
$(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS)
559568

Diff for: ‎examples/batched-bench/README.md

+48
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# llama.cpp/example/batched-bench
2+
3+
Benchmark the batched decoding performance of `llama.cpp`
4+
5+
## Usage
6+
7+
There are 2 modes of operation:
8+
9+
- `prompt not shared` - each batch has a separate prompt of size `PP` (i.e. `N_KV = B*(PP + TG)`)
10+
- `prompt is shared` - there is a common prompt of size `PP` used by all batches (i.e. `N_KV = PP + B*TG`)
11+
12+
```bash
13+
./batched-bench MODEL_PATH [N_KV_MAX] [IS_PP_SHARED] [NGL]
14+
15+
# LLaMA 7B, F16, N_KV_MAX = 16384 (8GB), prompt not shared
16+
./batched-bench ../models/llama-7b/ggml-model-f16.gguf 16384 0 99
17+
18+
# LLaMA 7B, Q8_0, N_KV_MAX = 16384 (8GB), prompt is shared
19+
./batched-bench ../models/llama-7b/ggml-model-q8_0.gguf 16384 1 99
20+
```
21+
22+
## Sample results
23+
24+
- `PP` - prompt tokens per batch
25+
- `TG` - generated tokens per batch
26+
- `B` - number of batches
27+
- `N_KV` - required KV cache size
28+
- `T_PP` - prompt processing time (i.e. time to first token)
29+
- `S_PP` - prompt processing speed (`(B*PP)/T_PP` or `PP/T_PP`)
30+
- `T_TG` - time to generate all batches
31+
- `S_TG` - text generation speed (`(B*TG)/T_TG`)
32+
- `T` - total time
33+
- `S` - total speed (i.e. all tokens / total time)
34+
35+
| PP | TG | B | N_KV | T_PP s | S_PP t/s | T_TG s | S_TG t/s | T s | S t/s |
36+
|-------|--------|------|--------|----------|----------|----------|----------|----------|----------|
37+
| 128 | 128 | 1 | 256 | 0.108 | 1186.64 | 3.079 | 41.57 | 3.187 | 80.32 |
38+
| 128 | 128 | 2 | 512 | 0.198 | 1295.19 | 5.029 | 50.90 | 5.227 | 97.95 |
39+
| 128 | 128 | 4 | 1024 | 0.373 | 1373.96 | 6.878 | 74.44 | 7.251 | 141.23 |
40+
| 128 | 128 | 8 | 2048 | 0.751 | 1363.27 | 7.344 | 139.43 | 8.095 | 252.99 |
41+
| 128 | 128 | 16 | 4096 | 1.570 | 1304.68 | 8.455 | 242.23 | 10.024 | 408.60 |
42+
| 128 | 128 | 32 | 8192 | 3.408 | 1201.73 | 8.801 | 465.40 | 12.209 | 670.96 |
43+
| 128 | 256 | 1 | 384 | 0.107 | 1196.70 | 6.329 | 40.45 | 6.436 | 59.67 |
44+
| 128 | 256 | 2 | 768 | 0.194 | 1317.45 | 10.239 | 50.00 | 10.433 | 73.61 |
45+
| 128 | 256 | 4 | 1536 | 0.366 | 1399.03 | 13.960 | 73.35 | 14.326 | 107.22 |
46+
| 128 | 256 | 8 | 3072 | 0.751 | 1363.92 | 15.110 | 135.54 | 15.861 | 193.69 |
47+
| 128 | 256 | 16 | 6144 | 1.569 | 1304.93 | 18.073 | 226.64 | 19.642 | 312.80 |
48+
| 128 | 256 | 32 | 12288 | 3.409 | 1201.35 | 19.223 | 426.15 | 22.633 | 542.93 |

Diff for: ‎examples/batched-bench/batched-bench.cpp

+13-10
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,11 @@ int main(int argc, char ** argv) {
1111
gpt_params params;
1212

1313
if (argc == 1 || argv[1][0] == '-') {
14-
printf("usage: %s MODEL_PATH [IS_PP_SHARED] [NGL]\n" , argv[0]);
14+
printf("usage: %s MODEL_PATH [N_KV_MAX] [IS_PP_SHARED] [NGL]\n" , argv[0]);
1515
return 1 ;
1616
}
1717

18+
int n_kv_max = 2048;
1819
int is_pp_shared = 0;
1920
int n_gpu_layers = 0;
2021

@@ -23,18 +24,20 @@ int main(int argc, char ** argv) {
2324
std::vector<int> n_pl = { 1, 2, 4, 8, 16, 32, };
2425
//std::vector<int> n_pl = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 32, };
2526

26-
const int32_t n_ctx_max = 16*1024;
27-
2827
if (argc >= 2) {
2928
params.model = argv[1];
3029
}
3130

3231
if (argc >= 3) {
33-
is_pp_shared = std::atoi(argv[2]);
32+
n_kv_max = std::atoi(argv[2]);
3433
}
3534

3635
if (argc >= 4) {
37-
n_gpu_layers = std::atoi(argv[3]);
36+
is_pp_shared = std::atoi(argv[3]);
37+
}
38+
39+
if (argc >= 5) {
40+
n_gpu_layers = std::atoi(argv[4]);
3841
}
3942

4043
// init LLM
@@ -56,8 +59,8 @@ int main(int argc, char ** argv) {
5659

5760
llama_context_params ctx_params = llama_context_default_params();
5861

59-
ctx_params.seed = 1234;
60-
ctx_params.n_ctx = n_ctx_max;
62+
ctx_params.seed = 1234;
63+
ctx_params.n_ctx = n_kv_max;
6164
ctx_params.n_batch = 512;
6265
ctx_params.n_threads = params.n_threads;
6366
ctx_params.n_threads_batch = params.n_threads_batch == -1 ? params.n_threads : params.n_threads_batch;
@@ -69,7 +72,7 @@ int main(int argc, char ** argv) {
6972
return 1;
7073
}
7174

72-
llama_batch batch = llama_batch_init(n_ctx_max, 0);
75+
llama_batch batch = llama_batch_init(n_kv_max, 0);
7376

7477
// decode in batches of ctx_params.n_batch tokens
7578
auto decode_helper = [](llama_context * ctx, llama_batch & batch, int32_t n_batch) {
@@ -88,7 +91,7 @@ int main(int argc, char ** argv) {
8891

8992
const int ret = llama_decode(ctx, batch_view);
9093
if (ret != 0) {
91-
LOG_TEE("%s : failed to decode the batch, n_batch = %d, ret = %d\n", __func__, n_batch, ret);
94+
LOG_TEE("failed to decode the batch, n_batch = %d, ret = %d\n", n_batch, ret);
9295
return false;
9396
}
9497
}
@@ -117,7 +120,7 @@ int main(int argc, char ** argv) {
117120

118121
const int n_ctx_req = is_pp_shared ? pp + pl*tg : pl*(pp + tg);
119122

120-
if (n_ctx_req > n_ctx_max) {
123+
if (n_ctx_req > n_kv_max) {
121124
continue;
122125
}
123126

0 commit comments

Comments
 (0)