diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index c4e86775b674..92af24238e21 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -849,11 +849,11 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr // mixed SWA/non-SWA layers with different n_dims or freq_base), we cannot // share a single precomputed rope_sin/rope_cos. Track divergence so the // translator falls back to per-op make_sin_cos in that case. - static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 15, "rope_params size"); + static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 16, "rope_params size"); if (!rope_seen) { - memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 15); + memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 16); rope_seen = true; - } else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 15) != 0) { + } else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 16) != 0) { model_params.mixed_rope_params = true; } } diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index bfa83917f423..183aa4cd5bd2 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -28,7 +28,7 @@ struct ModelParams { std::map n_heads_kv_per_layer; int head_size = -1; int state_size = -1; // for SSM molels, eg qwen35 - int32_t rope_params[15]; + int32_t rope_params[16]; bool mixed_rope_params = false; std::vector swa_layers; // The sliding-window mask tensor, identified in compute_llm_params() by grouping attention @@ -41,7 +41,7 @@ struct ModelParams { bool same_rope_params(const ModelParams & other) const { return mixed_rope_params == other.mixed_rope_params && - memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0; + memcmp(rope_params, other.rope_params, sizeof(int32_t) * 16) == 0; } bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other); } diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index cc8c85488568..9e9d31760bf4 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -1267,8 +1267,11 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { break; } case GGML_OP_CPY: { - if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16) { - return {false, "CPY with BF16 src type is not supported"}; + if (op->src[0]->type != GGML_TYPE_BF16 && op->src[1]->type == GGML_TYPE_BF16) { + return {false, "CPY with BF16 src[1] type is not supported"}; + } + if (ggml_openvino_get_device_name() == "NPU" && (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16)) { + return {false, "CPY with BF16 is not supported is not supported on NPU"}; } // CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend. if (ggml_is_quantized(op->type)) { @@ -1329,36 +1332,34 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { const int32_t * op_params = op->op_params; const int n_dims = op_params[1]; const int mode = op_params[2]; - if (op_params[15] != 0) { - // FIXME: support ggml_rope_set_offset - return {false, "ggml_rope_set_offset is not supported"}; - } + const int64_t n_offs = op_params[15]; if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_IMROPE) { return {false, "ROPE with mode " + std::to_string(mode) + " is not supported"}; } + if (n_offs < 0 || (n_offs % 2) != 0) { + return {false, "ROPE with invalid n_offs=" + std::to_string(n_offs)}; + } const int64_t head_dim = op->src[0]->ne[0]; const int64_t rope_dims = n_dims == 0 ? head_dim : n_dims; - if (rope_dims <= 0 || rope_dims > head_dim || (rope_dims % 2) != 0) { - return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", head_dim=" + std::to_string(head_dim) + " is not supported"}; + if (rope_dims <= 0 || rope_dims + n_offs > head_dim || (rope_dims % 2) != 0) { + return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", n_offs=" + std::to_string(n_offs) + + ", head_dim=" + std::to_string(head_dim) + " is not supported"}; } if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) { return {false, "ROPE with type " + std::string(ggml_type_name(op->type)) + " is not supported"}; } - if (op->src[0]->op == GGML_OP_VIEW) { - const struct ggml_tensor * view = op->src[0]; - const struct ggml_tensor * view_src = view->view_src; - if (view_src->ne[1] != view->ne[1] || view_src->ne[2] != view->ne[2] || view_src->ne[3] != view->ne[3]) { - return {false, "ROPE with view_src->ne [" + std::to_string(view_src->ne[1]) + ", " + - std::to_string(view_src->ne[2]) + ", " + std::to_string(view_src->ne[3]) + - "] != view->ne [" + std::to_string(view->ne[1]) + ", " + - std::to_string(view->ne[2]) + ", " + std::to_string(view->ne[3]) + - "] is not supported"}; - } + if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) { + return {false, "ROPE on VIEW / non-contiguous input is not supported"}; } + float freq_scale; + float ext_factor; + float attn_factor; + memcpy(&freq_scale, op_params + 6, sizeof(float)); + memcpy(&ext_factor, op_params + 7, sizeof(float)); + memcpy(&attn_factor, op_params + 8, sizeof(float)); if (mode == GGML_ROPE_TYPE_IMROPE && - (op->src[2] != 0 || ((const float *) op_params)[6] != 1 || ((const float *) op_params)[7] != 0 || - ((const float *) op_params)[8] != 1)) { - return {false, "IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor is not supported"}; + (op->src[2] != nullptr || freq_scale != 1.0f || ext_factor != 0.0f || attn_factor != 1.0f)) { + return {false, "IMROPE with freq_factors, freq_scale, ext_factor, or attn_factor is not supported"}; } break; } diff --git a/ggml/src/ggml-openvino/openvino/op/rope.cpp b/ggml/src/ggml-openvino/openvino/op/rope.cpp index 8f20a0d196eb..7e0d451fc6ed 100644 --- a/ggml/src/ggml-openvino/openvino/op/rope.cpp +++ b/ggml/src/ggml-openvino/openvino/op/rope.cpp @@ -44,6 +44,7 @@ OutputVector translate_rope(const NodeContext & context) { const int64_t head_dim = static_cast(output_shape[3]); const int64_t configured_n_dims = static_cast(op_params[1]); const int64_t n_dims = configured_n_dims == 0 ? head_dim : configured_n_dims; + const int64_t n_offs = static_cast(op_params[15]); constexpr int TYPE_NORMAL = 0; constexpr int TYPE_NEOX = 1; @@ -84,8 +85,10 @@ OutputVector translate_rope(const NodeContext & context) { data_node = std::make_shared(data_node, ov::element::f32); } - FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && n_dims <= head_dim && (n_dims % 2 == 0), - "ROPE expects even n_dims in [1, head_dim]"); + FRONT_END_OP_CONVERSION_CHECK(n_offs >= 0 && (n_offs % 2 == 0), + "ROPE expects non-negative even n_offs"); + FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && n_dims + n_offs <= head_dim && (n_dims % 2 == 0), + "ROPE expects even n_dims in [1, head_dim - n_offs]"); // TODO(openvino-gpu-rope-fusion): TEMPORARY WORKAROUND - do NOT revert until the // OpenVINO GPU plugin is updated. @@ -102,7 +105,6 @@ OutputVector translate_rope(const NodeContext & context) { // the active Flux rewrite here and the previous translation preserved below. if (mode == TYPE_NORMAL) { auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); // Emit the Flux-style interleaved-RoPE pattern so the GPU plugin's @@ -112,7 +114,7 @@ OutputVector translate_rope(const NodeContext & context) { // x1_neg = x1 * -1 // x_rotated = Reshape(Concat([x1_neg, x0], axis=-1), [1, S, n_heads, n_dims]) // y_rot = x_rot * t_cos + x_rotated * t_sin - // y = Concat([y_rot, x_tail], axis=-1) if n_dims < head_dim + // y = Concat([x_head, y_rot, x_tail], axis=-1) // Mathematically equivalent to the even/odd Slice form below. // // RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin @@ -128,8 +130,9 @@ OutputVector translate_rope(const NodeContext & context) { } const int64_t n_heads = static_cast(output_shape[2]); const int64_t half = n_dims / 2; - auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims}); - auto rot_data = std::make_shared(data_node, zero, rot_end, step_one, axis_last); + auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); + auto rot_data = std::make_shared(data_node, rot_start, rot_end, step_one, axis_last); auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); @@ -170,13 +173,24 @@ OutputVector translate_rope(const NodeContext & context) { auto y2 = std::make_shared(x_rotated, sin_full); auto rotated = std::make_shared(y1, y2); - if (n_dims < head_dim) { - auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims}); + ov::OutputVector concat_parts; + if (n_offs > 0) { + auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); + auto head = std::make_shared(data_node, head_start, head_end, step_one, axis_last); + concat_parts.push_back(head); + } + concat_parts.push_back(rotated); + if (n_offs + n_dims < head_dim) { + auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); auto tail = std::make_shared(data_node, tail_start, tail_end, step_one, axis_last); - res = std::make_shared(ov::OutputVector{rotated, tail}, -1); - } else { + concat_parts.push_back(tail); + } + if (concat_parts.size() == 1) { res = rotated; + } else { + res = std::make_shared(concat_parts, -1); } } // PRESERVED PREVIOUS TRANSLATION - Re-enable this branch (and remove the Flux branch above) once @@ -232,16 +246,26 @@ OutputVector translate_rope(const NodeContext & context) { data_node = std::make_shared(data_node, r4_shape, false); } auto axis_last = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}); - std::vector split_lengths = {n_dims / 2, n_dims / 2}; - if (n_dims < head_dim) { - split_lengths.push_back(head_dim - n_dims); + std::vector split_lengths; + if (n_offs > 0) { + split_lengths.push_back(n_offs); + } + split_lengths.push_back(n_dims / 2); + split_lengths.push_back(n_dims / 2); + if (n_offs + n_dims < head_dim) { + split_lengths.push_back(head_dim - (n_offs + n_dims)); } auto data_split = std::make_shared( data_node, axis_last, ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths)); - Output slice_data_node_0 = data_split->outputs()[0]; - Output slice_data_node_1 = data_split->outputs()[1]; + size_t split_idx = 0; + Output head_node; + if (n_offs > 0) { + head_node = data_split->outputs()[split_idx++]; + } + Output slice_data_node_0 = data_split->outputs()[split_idx++]; + Output slice_data_node_1 = data_split->outputs()[split_idx++]; auto first_half_node = std::make_shared( std::make_shared(slice_data_node_0, cos_theta_node), @@ -251,12 +275,16 @@ OutputVector translate_rope(const NodeContext & context) { std::make_shared(slice_data_node_0, sin_theta_node), std::make_shared(slice_data_node_1, cos_theta_node)); - if (n_dims < head_dim) { - Output tail = data_split->outputs()[2]; - res = std::make_shared(ov::OutputVector{first_half_node, second_half_node, tail}, -1); - } else { - res = std::make_shared(ov::OutputVector{first_half_node, second_half_node}, -1); + ov::OutputVector concat_parts; + if (n_offs > 0) { + concat_parts.push_back(head_node); + } + concat_parts.push_back(first_half_node); + concat_parts.push_back(second_half_node); + if (n_offs + n_dims < head_dim) { + concat_parts.push_back(data_split->outputs()[split_idx++]); } + res = std::make_shared(concat_parts, -1); } else if (mode == TYPE_IMROPE) { auto cos_sin_shape = std::make_shared(ov::element::i64, ov::Shape{4}, std::vector{1, -1, 1, (n_dims >> 1)}); @@ -264,16 +292,26 @@ OutputVector translate_rope(const NodeContext & context) { auto sin_reshaped = std::make_shared(sin_theta_node, cos_sin_shape, true); auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {3}); - std::vector split_lengths = {n_dims / 2, n_dims / 2}; - if (n_dims < head_dim) { - split_lengths.push_back(head_dim - n_dims); + std::vector split_lengths; + if (n_offs > 0) { + split_lengths.push_back(n_offs); + } + split_lengths.push_back(n_dims / 2); + split_lengths.push_back(n_dims / 2); + if (n_offs + n_dims < head_dim) { + split_lengths.push_back(head_dim - (n_offs + n_dims)); } auto split_a = std::make_shared( data_node, split_axis, ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths)); - auto x0 = split_a->output(0); - auto x1 = split_a->output(1); + size_t split_idx = 0; + Output head_node; + if (n_offs > 0) { + head_node = split_a->output(split_idx++); + } + auto x0 = split_a->output(split_idx++); + auto x1 = split_a->output(split_idx++); auto mul_a = std::make_shared(x0, cos_reshaped); auto mul_b = std::make_shared(x1, sin_reshaped); auto sub = std::make_shared(mul_a, mul_b); @@ -282,12 +320,16 @@ OutputVector translate_rope(const NodeContext & context) { auto mul_d = std::make_shared(x1, cos_reshaped); auto add = std::make_shared(mul_c, mul_d); - if (n_dims < head_dim) { - auto tail = split_a->output(2); - res = std::make_shared(ov::OutputVector{sub, add, tail}, 3); - } else { - res = std::make_shared(ov::OutputVector{sub, add}, 3); + ov::OutputVector concat_parts; + if (n_offs > 0) { + concat_parts.push_back(head_node); + } + concat_parts.push_back(sub); + concat_parts.push_back(add); + if (n_offs + n_dims < head_dim) { + concat_parts.push_back(split_a->output(split_idx++)); } + res = std::make_shared(concat_parts, 3); } if (res.get_element_type() != output_type) { diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 0e74c2cd5e87..09f73b53611a 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -445,7 +445,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< if (!model_cache_dir.empty() && !model_is_splitted) { const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful); model_fp = ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params, - 15, extra_cfg); + 16, extra_cfg); blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp); manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp);