From 5296b19a8559e05e4cf0ae5116b6a5d9a7263bfa Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Fri, 9 Jan 2026 11:29:40 -0800 Subject: [PATCH 1/2] npu perf fix --- ggml/src/ggml-openvino/ggml-openvino-extra.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 35d3d93cfd1..b6fec855fc8 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -165,7 +165,7 @@ clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL() { // Get requantization type for a tensor type (returns nullopt if no requant needed) std::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor) { if (strncmp(tensor->name, "token_embd.weight", 17) == 0) { - return ExtraQuantType::Q8_0_C; + return (ggml_openvino_is_npu() ? ExtraQuantType::F16 : ExtraQuantType::Q8_0_C); } if (strncmp(tensor->name, "output.weight", 13) == 0) { return ExtraQuantType::Q8_0_C; From 955e19136f734bace1f768a915f6ba3fdc500235 Mon Sep 17 00:00:00 2001 From: Mustafa Cavus Date: Mon, 12 Jan 2026 10:47:16 -0800 Subject: [PATCH 2/2] requant to f16 for Q6 embed on NPU --- ggml/src/ggml-openvino/ggml-openvino-extra.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index b6fec855fc8..bc0362ee460 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -165,7 +165,7 @@ clEnqueueMemcpyINTEL_fn ggml_openvino_get_clEnqueueMemcpyINTEL() { // Get requantization type for a tensor type (returns nullopt if no requant needed) std::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor) { if (strncmp(tensor->name, "token_embd.weight", 17) == 0) { - return (ggml_openvino_is_npu() ? ExtraQuantType::F16 : ExtraQuantType::Q8_0_C); + return ((ggml_openvino_is_npu() && tensor->type == GGML_TYPE_Q6_K) ? ExtraQuantType::F16 : ExtraQuantType::Q8_0_C); } if (strncmp(tensor->name, "output.weight", 13) == 0) { return ExtraQuantType::Q8_0_C;