Skip to content

Commit f6d7802

Browse files
cavusmustafacavusmustafa
committed
kvcachefusion support
1 parent 847da1a commit f6d7802

File tree

8 files changed

+146
-56
lines changed

8 files changed

+146
-56
lines changed

‎ggml/src/ggml-openvino/ggml-decoder.cpp‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -316,9 +316,13 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor* src) co
316316
input_shape = ov::PartialShape{1, -1, -1};
317317
}
318318
} else if (name.find("cache_") == 0) {
319-
int layer = extract_layer_from_name(name);
320-
bool is_swa = is_swa_layer(layer);
321-
input_shape = ov::PartialShape{is_swa ? m_context_size_swa : m_context_size, m_num_heads_kv, m_head_size};
319+
if (m_is_static) {
320+
int layer = extract_layer_from_name(name);
321+
bool is_swa = is_swa_layer(layer);
322+
input_shape = ov::PartialShape{is_swa ? m_context_size_swa : m_context_size, m_num_heads_kv, m_head_size};
323+
} else {
324+
input_shape = ov::PartialShape{1, -1, m_num_heads_kv, m_head_size};
325+
}
322326
} else if (const auto* op = get_tensor_used_op(src); op && op->op == GGML_OP_SET_ROWS) {
323327
input_shape = ov::PartialShape{1, 1, m_is_static ? 1 : -1};
324328
} else if (src->op == GGML_OP_VIEW) {

‎ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp‎

Lines changed: 46 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
#include <openvino/op/broadcast.hpp>
33
#include <openvino/op/concat.hpp>
44
#include <openvino/op/convert.hpp>
5+
#include <openvino/op/gather.hpp>
56
#include <openvino/op/reshape.hpp>
67
#include <openvino/op/scaled_dot_product_attention.hpp>
78
#include <openvino/op/transpose.hpp>
@@ -32,55 +33,82 @@ OutputVector translate_flash_attn_ext(const NodeContext& context) {
3233
auto q = std::make_shared<ov::op::v0::Convert>(q_f32, ov::element::f16);
3334
auto scale_node = std::make_shared<ov::op::v0::Constant>(ov::element::f16, ov::Shape{}, std::vector<float>{scale});
3435

35-
ov::Output<ov::Node> mask_sliced;
36+
ov::Output<ov::Node> mask_sliced, res;
3637
std::string mask_name = "KQ_mask_sliced";
3738
if (context.get_input_names()[3].find("swa") != std::string::npos) {
3839
mask_name = "KQ_mask_swa_sliced";
3940
}
4041
if (context.has_input(mask_name)) {
4142
mask_sliced = context.get_input(mask_name);
4243
} else {
43-
auto token_len = get_dimensions(q, {1});
44-
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
45-
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
46-
mask_sliced = std::make_shared<ov::op::v8::Slice>(mask, zero, token_len, one, one);
44+
auto token_len = get_dimensions(q, {2});
45+
auto zero_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {0,0});
46+
auto one_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,1});
47+
auto zero_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
48+
auto two_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
49+
auto axes = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,2});
50+
auto leaf_8 = context.get_input("leaf_8");
51+
auto shape_of_leaf_8 = std::make_shared<ov::op::v3::ShapeOf>(leaf_8);
52+
auto gather_leaf_8 = std::make_shared<ov::op::v8::Gather>(shape_of_leaf_8, two_1d, zero_1d);
53+
auto stop = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{token_len, gather_leaf_8}, 0);
54+
mask_sliced =
55+
std::make_shared<ov::op::v8::Slice>(mask, zero_2d, stop, one_2d, axes);
56+
mask_sliced = std::make_shared<ov::op::v0::Unsqueeze>(mask_sliced, zero_1d);
4757
}
4858

4959
if (mask_sliced.get_element_type() != ov::element::f16) {
5060
mask_sliced = std::make_shared<ov::op::v0::Convert>(mask_sliced, ov::element::f16);
5161
}
5262

53-
auto tile_kv = [](int64_t q_batch, int64_t kv_batch, ov::Output<Node> kv) {
63+
auto tile_kv = [](int64_t q_batch, int64_t kv_batch, ov::Output<Node> kv, bool is_static) {
5464
int64_t factor = q_batch / kv_batch;
5565
if (factor > 1) {
5666
auto q_batch_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector<int64_t>{q_batch});
5767
auto kv_batch_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector<int64_t>{kv_batch});
5868
auto factor_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector<int64_t>{factor});
5969

60-
auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {1});
61-
auto kv_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(kv, unsqueeze_axes);
70+
ov::Output<ov::Node> kv_broadcast_shape, kv_unsqueezed, new_kv_shape;
71+
if (is_static) {
72+
auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {1});
73+
kv_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(kv, unsqueeze_axes);
6274

63-
auto kv_last_two_dims = get_dimensions(kv.get_node_shared_ptr(), {1, 2});
64-
auto kv_broadcast_shape =
65-
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{kv_batch_node, factor_node, kv_last_two_dims}, 0);
66-
kv = std::make_shared<ov::op::v3::Broadcast>(kv_unsqueezed, kv_broadcast_shape);
75+
auto kv_last_two_dims = get_dimensions(kv.get_node_shared_ptr(), {1, 2});
76+
kv_broadcast_shape =
77+
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{kv_batch_node, factor_node, kv_last_two_dims}, 0);
78+
new_kv_shape =
79+
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{q_batch_node, kv_last_two_dims}, 0);
80+
} else {
81+
auto one_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
82+
auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2});
83+
kv_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(kv, unsqueeze_axes);
84+
85+
auto kv_last_two_dims = get_dimensions(kv.get_node_shared_ptr(), {2, 3});
86+
kv_broadcast_shape =
87+
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{one_1d, kv_batch_node, factor_node, kv_last_two_dims}, 0);
88+
new_kv_shape =
89+
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{one_1d, q_batch_node, kv_last_two_dims}, 0);
90+
}
6791

68-
auto new_kv_shape =
69-
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{q_batch_node, kv_last_two_dims}, 0);
92+
kv = std::make_shared<ov::op::v3::Broadcast>(kv_unsqueezed, kv_broadcast_shape);
7093
kv = std::make_shared<ov::op::v1::Reshape>(kv, new_kv_shape, false);
7194
}
7295
return kv;
7396
};
7497

7598
auto q_shape = context.get_input_shape(0).to_shape();
7699
auto k_shape = context.get_input_shape(1).to_shape();
77-
k = tile_kv(q_shape[0], k_shape[0], k);
78-
v = tile_kv(q_shape[0], k_shape[0], v);
100+
k = tile_kv(q_shape[0], k_shape[0], k, context.is_static());
101+
v = tile_kv(q_shape[0], k_shape[0], v, context.is_static());
79102

80103
auto sdpa = std::make_shared<ov::op::v13::ScaledDotProductAttention>(q, k, v, mask_sliced, scale_node, false);
81104
auto sdpa_f32 = std::make_shared<ov::op::v0::Convert>(sdpa, ov::element::f32);
82-
auto res = std::make_shared<ov::op::v1::Transpose>(sdpa_f32,
83-
ov::op::v0::Constant::create(ov::element::i64, {3}, {1, 0, 2}));
105+
if (context.is_static()) {
106+
res = std::make_shared<ov::op::v1::Transpose>(sdpa_f32,
107+
ov::op::v0::Constant::create(ov::element::i64, {3}, {1, 0, 2}));
108+
} else {
109+
res = std::make_shared<ov::op::v1::Transpose>(sdpa_f32,
110+
ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
111+
}
84112
return rename_outputs_with_suffix({res}, context.get_name());
85113
}
86114

‎ggml/src/ggml-openvino/openvino/op/mulmat.cpp‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,23 @@ OutputVector translate_mulmat(const NodeContext& context) {
5959

6060
auto Z_last_two_dims = get_dimensions(Z.get_node_shared_ptr(), {1, 2});
6161

62-
auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {1});
63-
auto Z_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(Z, unsqueeze_axes);
64-
6562
Output<Node> batch_small = A_batch_larger ? B_batch_node : A_batch_node;
6663
Output<Node> batch_large = A_batch_larger ? A_batch_node : B_batch_node;
67-
auto broadcast_shape =
68-
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{batch_small, factor_node, Z_last_two_dims}, 0);
64+
65+
ov::Output<Node> broadcast_shape;
66+
ov::Output<Node> Z_unsqueezed;
67+
if (context.is_static()) {
68+
auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {1});
69+
Z_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(Z, unsqueeze_axes);
70+
broadcast_shape =
71+
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{batch_small, factor_node, Z_last_two_dims}, 0);
72+
} else {
73+
auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2});
74+
Z_unsqueezed = std::make_shared<ov::op::v0::Unsqueeze>(Z, unsqueeze_axes);
75+
auto one_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
76+
broadcast_shape =
77+
std::make_shared<ov::op::v0::Concat>(ov::OutputVector{one_1d, batch_small, factor_node, Z_last_two_dims}, 0);
78+
}
6979
auto Z_broadcasted = std::make_shared<ov::op::v3::Broadcast>(Z_unsqueezed, broadcast_shape);
7080

7181
auto new_Z_shape = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{batch_large, Z_last_two_dims}, 0);

‎ggml/src/ggml-openvino/openvino/op/permute.cpp‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,13 @@ OutputVector translate_permute(const NodeContext& context) {
2525
ov::Output<Node> res;
2626

2727
if (op_case == 1) {
28-
res = std::make_shared<ov::op::v1::Transpose>(context.get_input(0),
29-
ov::op::v0::Constant::create(ov::element::i64, {3}, {1, 0, 2}));
28+
if (context.is_static()) {
29+
res = std::make_shared<ov::op::v1::Transpose>(context.get_input(0),
30+
ov::op::v0::Constant::create(ov::element::i64, {3}, {1, 0, 2}));
31+
} else {
32+
res = std::make_shared<ov::op::v1::Transpose>(context.get_input(0),
33+
ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
34+
}
3035
} else {
3136
auto src = context.get_input(0);
3237
Output<Node> attention_size;
@@ -38,20 +43,23 @@ OutputVector translate_permute(const NodeContext& context) {
3843
attention_size = context.get_input("attention_size_swa");
3944
}
4045

41-
auto src_shape_ = context.get_input_shape(0).to_shape();
42-
std::vector<int64_t> src_shape(src_shape_.begin(), src_shape_.end());
43-
44-
auto src_reshaped = std::make_shared<ov::op::v1::Reshape>(
45-
src,
46-
ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{-1, src_shape[1], src_shape[2]}),
47-
false);
48-
4946
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
5047
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
51-
auto src_slice = std::make_shared<ov::op::v8::Slice>(src_reshaped, zero, attention_size, one, zero);
5248

53-
res = std::make_shared<ov::op::v1::Transpose>(src_slice,
54-
ov::op::v0::Constant::create(ov::element::i64, {3}, {1, 0, 2}));
49+
if (context.is_static()) {
50+
auto src_shape_ = context.get_input_shape(0).to_shape();
51+
std::vector<int64_t> src_shape(src_shape_.begin(), src_shape_.end());
52+
auto src_reshaped = std::make_shared<ov::op::v1::Reshape>(
53+
src,
54+
ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector<int64_t>{-1, src_shape[1], src_shape[2]}),
55+
false);
56+
auto src_slice = std::make_shared<ov::op::v8::Slice>(src_reshaped, zero, attention_size, one, zero);
57+
res = std::make_shared<ov::op::v1::Transpose>(src_slice,
58+
ov::op::v0::Constant::create(ov::element::i64, {3}, {1, 0, 2}));
59+
} else {
60+
res = std::make_shared<ov::op::v1::Transpose>(src,
61+
ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}));
62+
}
5563
}
5664
return rename_outputs_with_suffix({res}, context.get_name());
5765
}

‎ggml/src/ggml-openvino/openvino/op/rope.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,9 @@ OutputVector translate_rope(const NodeContext& context) {
8484
ov::op::v0::Constant::create(ov::element::i64, {1}, {3}));
8585
auto stack = std::make_shared<ov::op::v0::Concat>(OutputVector{first_half, second_half}, 3);
8686
res = std::make_shared<ov::op::v1::Reshape>(stack, std::make_shared<ov::op::v0::ShapeOf>(data_node), false);
87+
if (!(context.is_static())) {
88+
res = std::make_shared<ov::op::v0::Unsqueeze>(res, ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
89+
}
8790
} else if (mode == ROPE_TYPE_NEOX) {
8891
auto data_split = std::make_shared<ov::op::v1::Split>(
8992
data_node, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2}), 2);

‎ggml/src/ggml-openvino/openvino/op/set_rows.cpp‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33
#include <openvino/core/node.hpp>
44
#include <openvino/core/node_output.hpp>
55
#include <openvino/frontend/exception.hpp>
6+
#include <openvino/op/concat.hpp>
67
#include <openvino/op/constant.hpp>
78
#include <openvino/op/convert.hpp>
89
#include <openvino/op/gather.hpp>
910
#include <openvino/op/reshape.hpp>
11+
#include <openvino/op/unsqueeze.hpp>
1012
#include <openvino/op/scatter_update.hpp>
1113
#include <openvino/op/shape_of.hpp>
1214
#include <openvino/op/slice.hpp>
@@ -39,17 +41,29 @@ OutputVector translate_set_rows(const NodeContext& context) {
3941
auto dst = context.get_input(context.get_output_name());
4042

4143
auto zero = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {0});
42-
auto dst_reshaped = std::make_shared<ov::op::v1::Reshape>(
43-
dst,
44-
ov::op::v0::Constant::create(ov::element::i64, {2}, {(int64_t) dst_shape[1], (int64_t) dst_shape[2]}),
45-
false);
46-
auto indices_reshaped =
47-
std::make_shared<ov::op::v0::Squeeze>(indices, ov::op::v0::Constant::create(ov::element::i64, {2}, {0, 1}));
48-
auto data_reshaped = std::make_shared<ov::op::v1::Reshape>(
49-
data, ov::op::v0::Constant::create(ov::element::i64, {2}, {(int64_t) -1, (int64_t) dst_shape[2]}), false);
50-
51-
auto updated = std::make_shared<ov::op::v3::ScatterUpdate>(dst_reshaped, indices_reshaped, data_reshaped, zero);
52-
auto res = std::make_shared<ov::op::v1::Reshape>(updated, std::make_shared<ov::op::v0::ShapeOf>(dst), false);
44+
Output<Node> res;
45+
if (context.is_static()) {
46+
auto dst_reshaped = std::make_shared<ov::op::v1::Reshape>(
47+
dst,
48+
ov::op::v0::Constant::create(ov::element::i64, {2}, {(int64_t) dst_shape[1], (int64_t) dst_shape[2]}),
49+
false);
50+
auto indices_reshaped =
51+
std::make_shared<ov::op::v0::Squeeze>(indices, ov::op::v0::Constant::create(ov::element::i64, {2}, {0, 1}));
52+
auto data_reshaped = std::make_shared<ov::op::v1::Reshape>(
53+
data, ov::op::v0::Constant::create(ov::element::i64, {2}, {(int64_t) -1, (int64_t) dst_shape[2]}), false);
54+
55+
auto updated = std::make_shared<ov::op::v3::ScatterUpdate>(dst_reshaped, indices_reshaped, data_reshaped, zero);
56+
res = std::make_shared<ov::op::v1::Reshape>(updated, std::make_shared<ov::op::v0::ShapeOf>(dst), false);
57+
} else {
58+
// TODO: Better solution would be to reshape the data into 4D at first place (for stateful model)
59+
if (data.get_partial_shape().rank() + 1 == dst.get_partial_shape().rank()) {
60+
data = std::make_shared<ov::op::v0::Unsqueeze>(data, zero);
61+
}
62+
int concat_axis = 1;
63+
if (context.is_static())
64+
concat_axis = 0;
65+
res = std::make_shared<ov::op::v0::Concat>(OutputVector{dst, data}, concat_axis);
66+
}
5367
return rename_outputs_with_suffix({res}, context.get_name());
5468
}
5569

‎ggml/src/ggml-openvino/openvino/op/softmax.cpp‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@
77
#include <openvino/op/concat.hpp>
88
#include <openvino/op/constant.hpp>
99
#include <openvino/op/convert.hpp>
10+
#include <openvino/op/gather.hpp>
1011
#include <openvino/op/matmul.hpp>
1112
#include <openvino/op/multiply.hpp>
13+
#include <openvino/op/unsqueeze.hpp>
1214
#include <openvino/op/slice.hpp>
1315
#include <openvino/op/softmax.hpp>
1416
#include <vector>
@@ -57,9 +59,20 @@ OutputVector translate_soft_max(const NodeContext& context) {
5759
} else {
5860
auto token_len = get_dimensions(input_node, {1});
5961
auto mask_node = context.get_input(1);
60-
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
61-
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
62-
mask_node_sliced = std::make_shared<ov::op::v8::Slice>(mask_node, zero, token_len, one, one);
62+
auto zero_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {0,0});
63+
auto one_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,1});
64+
auto zero_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
65+
auto two_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
66+
auto axes = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,2});
67+
auto leaf_8 = context.get_input("leaf_8");
68+
auto shape_of_leaf_8 = std::make_shared<ov::op::v3::ShapeOf>(leaf_8);
69+
auto gather_leaf_8 = std::make_shared<ov::op::v8::Gather>(shape_of_leaf_8, two_1d, zero_1d);
70+
auto stop = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{token_len, gather_leaf_8}, 0);
71+
mask_node_sliced =
72+
std::make_shared<ov::op::v8::Slice>(mask_node, zero_2d, stop, one_2d, axes);
73+
if (!(context.is_static())) {
74+
mask_node_sliced = std::make_shared<ov::op::v0::Unsqueeze>(mask_node_sliced, zero_1d);
75+
}
6376
}
6477

6578
if (mask_node_sliced.get_element_type() != context.get_output_type(0)) {

‎ggml/src/ggml-openvino/openvino/translate_session.cpp‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include <openvino/op/convert.hpp>
1212
#include <openvino/op/cos.hpp>
1313
#include <openvino/op/divide.hpp>
14+
#include <openvino/op/gather.hpp>
1415
#include <openvino/op/multiply.hpp>
1516
#include <openvino/op/parameter.hpp>
1617
#include <openvino/op/range.hpp>
@@ -87,9 +88,18 @@ void add_sliced_mask(TensorMap& tensor_map, GgmlDecoder& ggml_model_decoder) {
8788
if (is_static) {
8889
mask_sliced = mask;
8990
} else {
90-
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
91-
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
92-
mask_sliced = std::make_shared<ov::op::v8::Slice>(mask, zero, token_len, one, one);
91+
auto zero_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {0,0});
92+
auto one_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,1});
93+
auto zero_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
94+
auto two_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {2});
95+
auto axes = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,2});
96+
auto leaf_8 = tensor_map.at("leaf_8").get_node_shared_ptr();
97+
auto shape_of_leaf_8 = std::make_shared<ov::op::v3::ShapeOf>(leaf_8);
98+
auto gather_leaf_8 = std::make_shared<ov::op::v8::Gather>(shape_of_leaf_8, two_1d, zero_1d);
99+
auto stop = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{token_len, gather_leaf_8}, 0);
100+
mask_sliced =
101+
std::make_shared<ov::op::v8::Slice>(mask, zero_2d, stop, one_2d, axes);
102+
mask_sliced = std::make_shared<ov::op::v0::Unsqueeze>(mask_sliced, zero_1d);
93103
mask_sliced = std::make_shared<ov::op::v0::Convert>(mask_sliced, ov::element::f16);
94104
mask_sliced->set_friendly_name(sliced_name);
95105
}

0 commit comments

Comments
 (0)