Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 1 addition & 11 deletions ggml/src/ggml-openvino/ggml-decoder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -118,16 +118,6 @@ bool is_conv_states_all_tensor(const ggml_tensor * tensor) {
return tensor != nullptr && strncmp(tensor->name, "conv_states_all", strlen("conv_states_all")) == 0;
}

// CPY writing the tail of conv_input (the concat of the previous conv state and the new tokens)
// back into a slot block of the recurrent state cache. Detected structurally because the rollback
// variant (cparams.n_rs_seq > 0) emits one such CPY per snapshot slot without naming them.
bool is_conv_state_writeback(const ggml_tensor * node) {
return node->op == GGML_OP_CPY && node->view_src != nullptr && GgmlOvDecoder::is_kvcache(node->view_src, nullptr) &&
node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW &&
node->src[1]->view_src == node->view_src;
}

// MoE expert aggregation (build_moe_ffn in llama-graph.cpp): each expert plane is
// `ggml_view_2d(experts, n_embd, n_tokens, experts->nb[2], i*experts->nb[1])` and the planes
// are summed with a chain of ADDs: moe_out = ((view_0 + view_1) + view_2) + ... + view_{n-1}.
Expand Down Expand Up @@ -453,7 +443,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const {
if (node->src[0]->op == GGML_OP_VIEW) {
if (node->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET) {
op_case = 1;
} else if (is_conv_state_writeback(node)) {
} else if (GgmlOvDecoder::is_conv_state_writeback(node)) {
op_case = 2;
break;
} else if (is_conv_states_all_tensor(node->view_src) && node->src[1] != nullptr &&
Expand Down
7 changes: 7 additions & 0 deletions ggml/src/ggml-openvino/ggml-decoder.h
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,13 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder {
(op != nullptr && op->op == GGML_OP_SET_ROWS && op->src[2] == tensor);
}

inline static bool is_conv_state_writeback(const ggml_tensor * node) {
return node->op == GGML_OP_CPY && node->view_src != nullptr && is_kvcache(node->view_src, nullptr) &&
node->src[0] != nullptr && node->src[0]->op == GGML_OP_VIEW && node->src[0]->src[0] != nullptr &&
node->src[0]->src[0]->op == GGML_OP_CONCAT && node->src[1] != nullptr &&
node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src;
}

inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) {
return op->op == GGML_OP_SET_ROWS && op->src[1] == tensor;
}
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-openvino/ggml-openvino.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1059,7 +1059,7 @@ static bool cpy_output_view_is_supported(const ggml_tensor * op) {
return false;
}

return ggml_nbytes(op) == 0 || ggml_is_contiguous(op);
return ggml_nbytes(op) == 0 || ggml_is_contiguous(op) || GgmlOvDecoder::is_conv_state_writeback(op);
}

static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) {
Expand Down
Loading