Quant

ci/run.sh

@@ -103,7 +103,7 @@ if [ ! -z ${GG_BUILD_OPENVINO} ]; then
         echo "source /opt/intel/openvino/setupvars.sh"
         exit 1
     fi
-    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON"
+    CMAKE_EXTRA="${CMAKE_EXTRA} -DGGML_OPENVINO=ON -DGGML_CPU_REPACK=OFF"
 fi
 ## helpers
 

docs/build.md

@@ -648,7 +648,7 @@ git switch dev_backend_openvino
 
 # Build with OpenVINO support
 source /opt/intel/openvino/setupvars.sh
-cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON
+cmake -B build/ReleaseOV -G Ninja -DCMAKE_BUILD_TYPE=Release -DGGML_OPENVINO=ON -DGGML_CPU_REPACK=OFF
 cmake --build build/ReleaseOV --config Release -j $(nproc)
 ```
 

ggml/src/ggml-openvino/ggml-decoder.h

@@ -4,8 +4,10 @@
 #include <map>
 #include <memory>
 #include <openvino/core/partial_shape.hpp>
+#include <optional>
 #include <vector>
 
+#include "ggml-quants.hpp"
 #include "ggml.h"
 #include "openvino/decoder.hpp"
 
@@ -17,10 +19,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 
     // Node decoder, called in GgmlOvDecoder::visit_subgraph
     GgmlOvDecoder(struct ggml_tensor* node, struct ggml_cgraph* cgraph, bool is_static, bool is_first_token,
-                  int context_size, int num_heads, int num_heads_kv, int head_size);
+                  int context_size, int context_size_swa, int num_heads, int num_heads_kv, int head_size,
+                  const std::vector<int>& swa_layers);
 
     // Naive graph decoder
-    GgmlOvDecoder(struct ggml_cgraph* cgraph);
+    GgmlOvDecoder(struct ggml_cgraph* cgraph, std::map<std::string, std::shared_ptr<ov::Node>>& model_weights);
 
     virtual ov::Any get_attribute(const std::string& name) const override {
         return nullptr;
@@ -99,6 +102,12 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 
     virtual int get_context_size() const override { return m_context_size; }
 
+    virtual int get_context_size_swa() const override { return m_context_size_swa; }
+
+    virtual int is_swa_layer(int layer) const override {
+        return std::find(m_swa_layers.begin(), m_swa_layers.end(), layer) != m_swa_layers.end();
+    }
+
     virtual int get_num_heads() const override { return m_num_heads; }
 
     virtual int get_num_heads_kv() const override { return m_num_heads_kv; }
@@ -115,8 +124,12 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 
     ov::PartialShape get_graph_input_shape(const ggml_tensor* src) const;
 
-    static std::shared_ptr<ov::Node> create_weight_node(ggml_tensor* tensor);
-    static std::map<std::string, std::shared_ptr<ov::Node>> create_weight_nodes(struct ggml_cgraph* cgraph);
+    static void dump_cgraph(const struct ggml_cgraph* cgraph, std::string& filename);
+
+    static std::shared_ptr<ov::Node> create_weight_node(ggml_tensor* tensor,
+                                                        std::optional<ExtraQuantType> requant_type = std::nullopt);
+    static std::map<std::string, std::shared_ptr<ov::Node>> create_weight_nodes(
+        struct ggml_cgraph* cgraph, std::map<ggml_type, ExtraQuantType> types_to_requantize = {});
 
     const ggml_tensor* get_tensor_used_op(const ggml_tensor* tensor) const;
     const ggml_tensor* get_tensor_from_name(const std::string& name) const;
@@ -126,7 +139,6 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
 private:
     void set_input_output(ggml_tensor* node, bool naive = false);
     void add_extra_inputs();
-    static void dump_cgraph(const struct ggml_cgraph* cgraph, std::string& filename);
     static std::vector<size_t> get_shape(const ggml_tensor* tensor);
     static std::vector<size_t> get_stride(const ggml_tensor* tensor);
     static ov::element::Type get_ov_type(const ggml_tensor* tensor);
@@ -151,13 +163,17 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
     std::map<std::string, std::shared_ptr<ov::Node>> m_model_weights;
     std::vector<std::string> m_model_output_names;
     int m_context_size;
+    int m_context_size_swa;
+    std::vector<int> m_swa_layers;
     int m_num_heads;
     int m_num_heads_kv;
     int m_head_size;
     int32_t* m_rope_params;
     std::vector<std::string> m_kv_names;
-    bool m_is_static;
+    bool m_is_static = false;
     bool m_is_first_token;
 };
 
 void print_tensor_address_map(const struct ggml_cgraph* cgraph);
+
+int extract_layer_from_name(const std::string& name);

ggml/src/ggml-openvino/ggml-openvino.cpp

@@ -8,6 +8,7 @@
 #include <vector>
 
 #include "ggml-backend-impl.h"
+#include "ggml-backend.h"
 #include "ggml-impl.h"
 #include "ggml-openvino/utils.h"
 #include "ggml.h"
@@ -248,17 +249,30 @@ static bool is_op_unsupported_case(const ggml_tensor* op) {
         const auto* op_params = op->op_params;
         memcpy(&scale, (const float*) op_params + 0, sizeof(float));
         memcpy(&max_bias, (const float*) op_params + 1, sizeof(float));
-        const uint32_t h = op->src[0]->ne[2];
-        const uint32_t n_head = op->src[0]->ne[0];
-        const uint32_t n_head_log2 = 1u << (uint32_t) floor(log2(n_head));
-
-        const float m0 = powf(2.0f, -(max_bias) / n_head_log2);
-        const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
-        const float slope =
-            (max_bias > 0.0f) ? h < n_head_log2 ? powf(m0, h + 1) : powf(m1, 2 * (h - n_head_log2) + 1) : 1.0f;
+        if (max_bias > 0) {
+            GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with max_bias > 0\n");
+            return true;
+        }
+    }
 
-        if (slope != 1.0f) {
-            GGML_LOG_WARN("OpenVINO backend does not support SOFT_MAX with slope != 1.0f\n");
+    if (op->op == GGML_OP_FLASH_ATTN_EXT) {
+        if (op->src[4] != nullptr) {
+            GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with sinks\n");
+            return true;
+        }
+        float scale = 1.0f;
+        float max_bias = 0.0f;
+        float logit_softcap = 0.0f;
+        const auto* op_params = op->op_params;
+        memcpy(&scale, (const float*) op_params + 0, sizeof(float));
+        memcpy(&max_bias, (const float*) op_params + 1, sizeof(float));
+        memcpy(&logit_softcap, (const float*) op_params + 2, sizeof(float));
+        if (max_bias > 0) {
+            GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with max_bias > 0\n");
+            return true;
+        }
+        if (logit_softcap != 0) {
+            GGML_LOG_WARN("OpenVINO backend does not support FLASH_ATTN_EXT with logit_softcap != 0\n");
             return true;
         }
     }
@@ -305,12 +319,8 @@ static bool is_op_unsupported_case(const ggml_tensor* op) {
             return true;
         }
         float freq_scale;
-        memcpy(&freq_scale, op_params + 6, sizeof(float));
-        if (freq_scale != 0.0f && freq_scale != 1.0f) {
-            GGML_LOG_WARN("OpenVINO backend does not support ROPE with freq_scale %f != 1.0f\n", freq_scale);
-            return true;
-        }
         float ext_factor;
+        memcpy(&freq_scale, op_params + 6, sizeof(float));
         memcpy(&ext_factor, op_params + 7, sizeof(float));
         if (ext_factor != 0.0f) {
             GGML_LOG_WARN("OpenVINO backend does not support ROPE with ext_factor %f != 0.0f\n", ext_factor);
@@ -332,8 +342,17 @@ static bool is_op_unsupported_case(const ggml_tensor* op) {
 static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, const ggml_tensor* op) {
     GGML_ASSERT(dev->reg != nullptr);
 
-    static const std::set<ggml_type> supported_types{
-        GGML_TYPE_F32, GGML_TYPE_F16, GGML_TYPE_BF16, GGML_TYPE_I64, GGML_TYPE_I32};
+    static std::set<ggml_type> supported_types{GGML_TYPE_F32,
+                                               GGML_TYPE_F16,
+                                               GGML_TYPE_BF16,
+                                               GGML_TYPE_I64,
+                                               GGML_TYPE_I32,
+                                               GGML_TYPE_Q4_0,
+                                               GGML_TYPE_Q4_1,
+                                               GGML_TYPE_Q4_K,
+                                               GGML_TYPE_Q5_K,
+                                               GGML_TYPE_Q8_0,
+                                               GGML_TYPE_Q6_K};
 
     static const std::set<ggml_op> supported_ops{GGML_OP_NONE,
                                                  GGML_OP_ADD,
@@ -348,7 +367,8 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
                                                  GGML_OP_ROPE,
                                                  GGML_OP_RMS_NORM,
                                                  GGML_OP_SCALE,
-                                                 GGML_OP_SOFT_MAX,
+                                                 // softmax is not updated due to replaced by flash_attn_ext
+                                                 // GGML_OP_SOFT_MAX,
                                                  GGML_OP_SET_ROWS,
                                                  GGML_OP_FLASH_ATTN_EXT,
                                                  GGML_OP_CPY};
@@ -357,6 +377,7 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
     };
     static const std::set<ggml_glu_op> supported_glu_ops{
         GGML_GLU_OP_SWIGLU,
+        GGML_GLU_OP_GEGLU,
     };
 
     switch (op->op) {
@@ -394,14 +415,22 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con
         return false;
     }
     for (int i = 0; i < GGML_MAX_SRC; i++) {
-        if (supported_types.find(op->type) == supported_types.end()) {
-            GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(op->type));
+        auto* src = op->src[i];
+        if (src == nullptr) {
+            break;
+        }
+        if (supported_types.find(src->type) == supported_types.end()) {
+            GGML_LOG_WARN("OpenVINO backend does not support tensor type %s\n", ggml_type_name(src->type));
             return false;
         }
-        if (op->src[i] != nullptr && op->src[i]->ne[3] != 1) {
+        if (src->ne[3] != 1) {
             GGML_LOG_WARN("OpenVINO backend does not support tensors with ne[3] != 1\n");
             return false;
         }
+        if (ggml_is_quantized(src->type) && src->ne[2] != 1) {
+            GGML_LOG_WARN("OpenVINO backend does not support 3D quantized tensors\n");
+            return false;
+        }
     }
 
     if (is_op_unsupported_case(op)) {