From 510b537d4289a79b898f8e7c88b3f85b8968c8e6 Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Mon, 19 Jun 2023 08:38:38 +0300 Subject: [PATCH 1/2] Only use Q6_K for output weights if tensor size is multiple of 256 --- llama.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/llama.cpp b/llama.cpp index 2105e32799ae9..87b86dc0c64bc 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2495,7 +2495,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (quantized_type == GGML_TYPE_Q2_K || quantized_type == GGML_TYPE_Q3_K || quantized_type == GGML_TYPE_Q4_K || quantized_type == GGML_TYPE_Q5_K || quantized_type == GGML_TYPE_Q6_K) { int nx = tensor.ne.at(0); - int ny = tensor.ne.at(0); + int ny = tensor.ne.at(1); if (nx % QK_K != 0 || ny % QK_K != 0) { fprintf(stderr, "\n\n========================= Tensor sizes %d x %d are not divisible by %d\n",nx,ny,QK_K); fprintf(stderr, "This is required to be able to use k-quants for now!\n"); @@ -2504,7 +2504,11 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s } } if (tensor.name == "output.weight") { - new_type = GGML_TYPE_Q6_K; + int nx = tensor.ne.at(0); + int ny = tensor.ne.at(1); + if (nx % QK_K == 0 || ny % QK_K == 0) { + new_type = GGML_TYPE_Q6_K; + } } else if (tensor.name.find("attention.wv.weight") != std::string::npos) { if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K; else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K; From 986a56e2b226d0e76df35421064321b649a0b804 Mon Sep 17 00:00:00 2001 From: Iwan Kawrakow Date: Mon, 19 Jun 2023 11:39:47 +0300 Subject: [PATCH 2/2] Fixed copy/paste mistake --- llama.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama.cpp b/llama.cpp index 87b86dc0c64bc..6bce66d161346 100644 --- a/llama.cpp +++ b/llama.cpp @@ -2506,7 +2506,7 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s if (tensor.name == "output.weight") { int nx = tensor.ne.at(0); int ny = tensor.ne.at(1); - if (nx % QK_K == 0 || ny % QK_K == 0) { + if (nx % QK_K == 0 && ny % QK_K == 0) { new_type = GGML_TYPE_Q6_K; } } else if (tensor.name.find("attention.wv.weight") != std::string::npos) {