Skip to content

ggml-openvino : support ggml_rope_set_offset and simplify op support gating #304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cavusmustafa merged 3 commits into from
Sep 4, 2026
Merged

ggml-openvino : support ggml_rope_set_offset and simplify op support gating #304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3f0c94c
ggml-openvino : support ggml_rope_set_offset and simplify op support …
mostafafaheem Aug 28, 2026
f00128c
add more cpy cases
mostafafaheem Aug 28, 2026
c3abdb3
reject BF16 cpy on NPU
mostafafaheem Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ggml/src/ggml-openvino/ggml-decoder.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -849,11 +849,11 @@ std::pair<ModelParams, ComputeParams> GgmlOvDecoder::compute_llm_params(ggml_cgr
// mixed SWA/non-SWA layers with different n_dims or freq_base), we cannot
// share a single precomputed rope_sin/rope_cos. Track divergence so the
// translator falls back to per-op make_sin_cos in that case.
static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 15, "rope_params size");
static_assert(sizeof(model_params.rope_params) == sizeof(int32_t) * 16, "rope_params size");
if (!rope_seen) {
memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 15);
memcpy(model_params.rope_params, node->op_params, sizeof(int32_t) * 16);
rope_seen = true;
} else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 15) != 0) {
} else if (memcmp(model_params.rope_params, node->op_params, sizeof(int32_t) * 16) != 0) {
model_params.mixed_rope_params = true;
}
}
Expand Down
4 changes: 2 additions & 2 deletions ggml/src/ggml-openvino/ggml-decoder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ struct ModelParams {
std::map<int, int> n_heads_kv_per_layer;
int head_size = -1;
int state_size = -1; // for SSM molels, eg qwen35
int32_t rope_params[15];
int32_t rope_params[16];
bool mixed_rope_params = false;
std::vector<int> swa_layers;
// The sliding-window mask tensor, identified in compute_llm_params() by grouping attention
Expand All @@ -41,7 +41,7 @@ struct ModelParams {

bool same_rope_params(const ModelParams & other) const {
return mixed_rope_params == other.mixed_rope_params &&
memcmp(rope_params, other.rope_params, sizeof(int32_t) * 15) == 0;
memcmp(rope_params, other.rope_params, sizeof(int32_t) * 16) == 0;
}

bool can_reuse_dynamically(const ModelParams & other) const { return same_rope_params(other); }
Expand Down
43 changes: 22 additions & 21 deletions ggml/src/ggml-openvino/ggml-openvino.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1267,8 +1267,11 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
break;
}
case GGML_OP_CPY: {
if (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16) {
return {false, "CPY with BF16 src type is not supported"};
if (op->src[0]->type != GGML_TYPE_BF16 && op->src[1]->type == GGML_TYPE_BF16) {
return {false, "CPY with BF16 src[1] type is not supported"};
}
if (ggml_openvino_get_device_name() == "NPU" && (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16)) {
return {false, "CPY with BF16 is not supported is not supported on NPU"};
}
// CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend.
if (ggml_is_quantized(op->type)) {
Expand Down Expand Up @@ -1329,36 +1332,34 @@ static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) {
const int32_t * op_params = op->op_params;
const int n_dims = op_params[1];
const int mode = op_params[2];
if (op_params[15] != 0) {
// FIXME: support ggml_rope_set_offset
return {false, "ggml_rope_set_offset is not supported"};
}
const int64_t n_offs = op_params[15];
if (mode != GGML_ROPE_TYPE_NORMAL && mode != GGML_ROPE_TYPE_NEOX && mode != GGML_ROPE_TYPE_IMROPE) {
return {false, "ROPE with mode " + std::to_string(mode) + " is not supported"};
}
if (n_offs < 0 || (n_offs % 2) != 0) {
return {false, "ROPE with invalid n_offs=" + std::to_string(n_offs)};
}
const int64_t head_dim = op->src[0]->ne[0];
const int64_t rope_dims = n_dims == 0 ? head_dim : n_dims;
if (rope_dims <= 0 || rope_dims > head_dim || (rope_dims % 2) != 0) {
return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", head_dim=" + std::to_string(head_dim) + " is not supported"};
if (rope_dims <= 0 || rope_dims + n_offs > head_dim || (rope_dims % 2) != 0) {
return {false, "ROPE with n_dims=" + std::to_string(n_dims) + ", n_offs=" + std::to_string(n_offs) +
", head_dim=" + std::to_string(head_dim) + " is not supported"};
}
if (op->type != GGML_TYPE_F32 && op->type != GGML_TYPE_F16) {
return {false, "ROPE with type " + std::string(ggml_type_name(op->type)) + " is not supported"};
}
if (op->src[0]->op == GGML_OP_VIEW) {
const struct ggml_tensor * view = op->src[0];
const struct ggml_tensor * view_src = view->view_src;
if (view_src->ne[1] != view->ne[1] || view_src->ne[2] != view->ne[2] || view_src->ne[3] != view->ne[3]) {
return {false, "ROPE with view_src->ne [" + std::to_string(view_src->ne[1]) + ", " +
std::to_string(view_src->ne[2]) + ", " + std::to_string(view_src->ne[3]) +
"] != view->ne [" + std::to_string(view->ne[1]) + ", " +
std::to_string(view->ne[2]) + ", " + std::to_string(view->ne[3]) +
"] is not supported"};
}
if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) {
return {false, "ROPE on VIEW / non-contiguous input is not supported"};
}
float freq_scale;
float ext_factor;
float attn_factor;
memcpy(&freq_scale, op_params + 6, sizeof(float));
memcpy(&ext_factor, op_params + 7, sizeof(float));
memcpy(&attn_factor, op_params + 8, sizeof(float));
if (mode == GGML_ROPE_TYPE_IMROPE &&
(op->src[2] != 0 || ((const float *) op_params)[6] != 1 || ((const float *) op_params)[7] != 0 ||
((const float *) op_params)[8] != 1)) {
return {false, "IMROPE with freq_factors, freq_scale, ext_factor, and attn_factor is not supported"};
(op->src[2] != nullptr || freq_scale != 1.0f || ext_factor != 0.0f || attn_factor != 1.0f)) {
return {false, "IMROPE with freq_factors, freq_scale, ext_factor, or attn_factor is not supported"};
}
break;
}
Expand Down
102 changes: 72 additions & 30 deletions ggml/src/ggml-openvino/openvino/op/rope.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ OutputVector translate_rope(const NodeContext & context) {
const int64_t head_dim = static_cast<int64_t>(output_shape[3]);
const int64_t configured_n_dims = static_cast<int64_t>(op_params[1]);
const int64_t n_dims = configured_n_dims == 0 ? head_dim : configured_n_dims;
const int64_t n_offs = static_cast<int64_t>(op_params[15]);

constexpr int TYPE_NORMAL = 0;
constexpr int TYPE_NEOX = 1;
Expand Down Expand Up @@ -84,8 +85,10 @@ OutputVector translate_rope(const NodeContext & context) {
data_node = std::make_shared<ov::op::v0::Convert>(data_node, ov::element::f32);
}

FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && n_dims <= head_dim && (n_dims % 2 == 0),
"ROPE expects even n_dims in [1, head_dim]");
FRONT_END_OP_CONVERSION_CHECK(n_offs >= 0 && (n_offs % 2 == 0),
"ROPE expects non-negative even n_offs");
FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && n_dims + n_offs <= head_dim && (n_dims % 2 == 0),
"ROPE expects even n_dims in [1, head_dim - n_offs]");

// TODO(openvino-gpu-rope-fusion): TEMPORARY WORKAROUND - do NOT revert until the
// OpenVINO GPU plugin is updated.
Expand All @@ -102,7 +105,6 @@ OutputVector translate_rope(const NodeContext & context) {
// the active Flux rewrite here and the previous translation preserved below.
if (mode == TYPE_NORMAL) {
auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});

// Emit the Flux-style interleaved-RoPE pattern so the GPU plugin's
Expand All @@ -112,7 +114,7 @@ OutputVector translate_rope(const NodeContext & context) {
// x1_neg = x1 * -1
// x_rotated = Reshape(Concat([x1_neg, x0], axis=-1), [1, S, n_heads, n_dims])
// y_rot = x_rot * t_cos + x_rotated * t_sin
// y = Concat([y_rot, x_tail], axis=-1) if n_dims < head_dim
// y = Concat([x_head, y_rot, x_tail], axis=-1)
// Mathematically equivalent to the even/odd Slice form below.
//
// RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin
Expand All @@ -128,8 +130,9 @@ OutputVector translate_rope(const NodeContext & context) {
}
const int64_t n_heads = static_cast<int64_t>(output_shape[2]);
const int64_t half = n_dims / 2;
auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims});
auto rot_data = std::make_shared<ov::op::v8::Slice>(data_node, zero, rot_end, step_one, axis_last);
auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs});
auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims});
auto rot_data = std::make_shared<ov::op::v8::Slice>(data_node, rot_start, rot_end, step_one, axis_last);

auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f});

Expand Down Expand Up @@ -170,13 +173,24 @@ OutputVector translate_rope(const NodeContext & context) {
auto y2 = std::make_shared<ov::op::v1::Multiply>(x_rotated, sin_full);
auto rotated = std::make_shared<ov::op::v1::Add>(y1, y2);

if (n_dims < head_dim) {
auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_dims});
ov::OutputVector concat_parts;
if (n_offs > 0) {
auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs});
auto head = std::make_shared<ov::op::v8::Slice>(data_node, head_start, head_end, step_one, axis_last);
concat_parts.push_back(head);
}
concat_parts.push_back(rotated);
if (n_offs + n_dims < head_dim) {
auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims});
auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim});
auto tail = std::make_shared<ov::op::v8::Slice>(data_node, tail_start, tail_end, step_one, axis_last);
res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{rotated, tail}, -1);
} else {
concat_parts.push_back(tail);
}
if (concat_parts.size() == 1) {
res = rotated;
} else {
res = std::make_shared<ov::op::v0::Concat>(concat_parts, -1);
}
}
// PRESERVED PREVIOUS TRANSLATION - Re-enable this branch (and remove the Flux branch above) once
Expand Down Expand Up @@ -232,16 +246,26 @@ OutputVector translate_rope(const NodeContext & context) {
data_node = std::make_shared<ov::op::v1::Reshape>(data_node, r4_shape, false);
}
auto axis_last = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1});
std::vector<int64_t> split_lengths = {n_dims / 2, n_dims / 2};
if (n_dims < head_dim) {
split_lengths.push_back(head_dim - n_dims);
std::vector<int64_t> split_lengths;
if (n_offs > 0) {
split_lengths.push_back(n_offs);
}
split_lengths.push_back(n_dims / 2);
split_lengths.push_back(n_dims / 2);
if (n_offs + n_dims < head_dim) {
split_lengths.push_back(head_dim - (n_offs + n_dims));
}

auto data_split = std::make_shared<ov::op::v1::VariadicSplit>(
data_node, axis_last,
ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths));
Output<Node> slice_data_node_0 = data_split->outputs()[0];
Output<Node> slice_data_node_1 = data_split->outputs()[1];
size_t split_idx = 0;
Output<Node> head_node;
if (n_offs > 0) {
head_node = data_split->outputs()[split_idx++];
}
Output<Node> slice_data_node_0 = data_split->outputs()[split_idx++];
Output<Node> slice_data_node_1 = data_split->outputs()[split_idx++];

auto first_half_node = std::make_shared<ov::op::v1::Subtract>(
std::make_shared<ov::op::v1::Multiply>(slice_data_node_0, cos_theta_node),
Expand All @@ -251,29 +275,43 @@ OutputVector translate_rope(const NodeContext & context) {
std::make_shared<ov::op::v1::Multiply>(slice_data_node_0, sin_theta_node),
std::make_shared<ov::op::v1::Multiply>(slice_data_node_1, cos_theta_node));

if (n_dims < head_dim) {
Output<Node> tail = data_split->outputs()[2];
res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{first_half_node, second_half_node, tail}, -1);
} else {
res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{first_half_node, second_half_node}, -1);
ov::OutputVector concat_parts;
if (n_offs > 0) {
concat_parts.push_back(head_node);
}
concat_parts.push_back(first_half_node);
concat_parts.push_back(second_half_node);
if (n_offs + n_dims < head_dim) {
concat_parts.push_back(data_split->outputs()[split_idx++]);
}
res = std::make_shared<ov::op::v0::Concat>(concat_parts, -1);
} else if (mode == TYPE_IMROPE) {
auto cos_sin_shape = std::make_shared<ov::op::v0::Constant>(ov::element::i64, ov::Shape{4},
std::vector<int64_t>{1, -1, 1, (n_dims >> 1)});
auto cos_reshaped = std::make_shared<ov::op::v1::Reshape>(cos_theta_node, cos_sin_shape, true);
auto sin_reshaped = std::make_shared<ov::op::v1::Reshape>(sin_theta_node, cos_sin_shape, true);

auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {3});
std::vector<int64_t> split_lengths = {n_dims / 2, n_dims / 2};
if (n_dims < head_dim) {
split_lengths.push_back(head_dim - n_dims);
std::vector<int64_t> split_lengths;
if (n_offs > 0) {
split_lengths.push_back(n_offs);
}
split_lengths.push_back(n_dims / 2);
split_lengths.push_back(n_dims / 2);
if (n_offs + n_dims < head_dim) {
split_lengths.push_back(head_dim - (n_offs + n_dims));
}

auto split_a = std::make_shared<ov::op::v1::VariadicSplit>(
data_node, split_axis,
ov::op::v0::Constant::create(ov::element::i64, {split_lengths.size()}, split_lengths));
auto x0 = split_a->output(0);
auto x1 = split_a->output(1);
size_t split_idx = 0;
Output<Node> head_node;
if (n_offs > 0) {
head_node = split_a->output(split_idx++);
}
auto x0 = split_a->output(split_idx++);
auto x1 = split_a->output(split_idx++);
auto mul_a = std::make_shared<ov::op::v1::Multiply>(x0, cos_reshaped);
auto mul_b = std::make_shared<ov::op::v1::Multiply>(x1, sin_reshaped);
auto sub = std::make_shared<ov::op::v1::Subtract>(mul_a, mul_b);
Expand All @@ -282,12 +320,16 @@ OutputVector translate_rope(const NodeContext & context) {
auto mul_d = std::make_shared<ov::op::v1::Multiply>(x1, cos_reshaped);
auto add = std::make_shared<ov::op::v1::Add>(mul_c, mul_d);

if (n_dims < head_dim) {
auto tail = split_a->output(2);
res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{sub, add, tail}, 3);
} else {
res = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{sub, add}, 3);
ov::OutputVector concat_parts;
if (n_offs > 0) {
concat_parts.push_back(head_node);
}
concat_parts.push_back(sub);
concat_parts.push_back(add);
if (n_offs + n_dims < head_dim) {
concat_parts.push_back(split_a->output(split_idx++));
}
res = std::make_shared<ov::op::v0::Concat>(concat_parts, 3);
}

if (res.get_element_type() != output_type) {
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-openvino/utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr<
if (!model_cache_dir.empty() && !model_is_splitted) {
const uint64_t extra_cfg = ggml_openvino_model_cache_extra_cfg(device, stateful);
model_fp = ggml_openvino_model_fingerprint(cgraph, device, /*fa=*/true, m_params.rope_params,
15, extra_cfg);
16, extra_cfg);
blob_path = ggml_openvino_model_cache_blob_path(model_cache_dir, model_fp);
manifest_path = ggml_openvino_model_cache_manifest_path(model_cache_dir, model_fp);

Expand Down
Loading