diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index e0183d5ac..4382aa6ec 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -300,6 +300,10 @@ def run_agentic_conversation( response_id: str | None = None conversation_tool_call_events: list[ToolCallEvent] = [] conversation_reasoning_step_events: list[ReasoningStepEvent] = [] + # Skills activated by any turn so far. The platform keeps a skill active across + # turns once set -- an agent correctly reuses the already-active skill without + # re-issuing set_skills, so routing credit must not require a fresh call every turn. + activated_skills_so_far: set[str] = set() # Every send_message() call (across every logical turn AND every clarification # sub-turn within it) restarts call_ts/ts near 0 -- these run across the whole # conversation, not reset per logical turn, so every one of those calls shifts them. @@ -375,7 +379,8 @@ def run_agentic_conversation( current_message = _get_sim_user_response(response_text, resolved_turn, resolved_expected) activated = _activated_skills(all_tool_calls) - skill_routing = turn.expected_skill in activated if activated else False + activated_skills_so_far |= set(activated) + skill_routing = turn.expected_skill in activated_skills_so_far output_present = _check_output_present(resolved_turn, final_result) if final_result else False output_correct = ( _check_output_correct(resolved_turn, final_result) if (final_result and output_present) else None diff --git a/packages/gooddata-eval/tests/test_agentic_conversation.py b/packages/gooddata-eval/tests/test_agentic_conversation.py index 144c432ed..fd7fa18d4 100644 --- a/packages/gooddata-eval/tests/test_agentic_conversation.py +++ b/packages/gooddata-eval/tests/test_agentic_conversation.py @@ -329,6 +329,64 @@ def test_run_agentic_conversation_deletes_every_unique_metric_across_turns(): assert deleted == [("ws1", "extra"), ("ws1", "shared")] +def test_run_agentic_conversation_skill_routing_persists_across_turns(): + """A skill activated in an earlier turn stays credited when a later turn reuses it + without re-issuing set_skills -- the platform keeps a skill active once set, so an + agent correctly omits a redundant set_skills call. Requiring a fresh call every turn + produced false FAILs on turns that did the right thing (found via + debug_conversation.py replaying analyst-explores-dynamic-currency-conversion, + turns t4/t5: create_adhoc_visualization/create_metric both ran and succeeded, but + skill_routing was False solely because set_skills wasn't repeated).""" + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.side_effect = [ + _metric_turn_result([_skills_tc("metric"), _create_metric_tc("m1")]), + _metric_turn_result([_create_metric_tc("m2")]), # no set_skills -- skill already active + ] + + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + ): + result = run_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=_two_metric_turn_fixture(), + ) + + assert result.turn_results[0].skill_routing is True + assert result.turn_results[1].skill_routing is True + + +def test_run_agentic_conversation_skill_routing_false_when_skill_never_activated(): + """Guard against the fix being too lenient: a skill that no turn ever activated + must still fail routing, not be credited by the cumulative-set change.""" + mock_client = MagicMock() + mock_client.create_conversation.return_value = "conv-1" + mock_client.send_message.return_value = _metric_turn_result([_create_metric_tc("m1")]) + + fixture = ConversationFixture( + id="test-never-activated", + expected_skills=["metric"], + turns=[ + TurnDefinition(turn_id="t1", message="Create x", expected_skill="metric", expected_output_type="metric"), + ], + ) + with ( + patch("gooddata_eval.core.agentic.conversation.ChatClient", return_value=mock_client), + patch("gooddata_eval.core.agentic.conversation.GoodDataSdk"), + ): + result = run_agentic_conversation( + host="http://host/api/v1/actions/workspaces/ws1/ai", + token="tok", + workspace_id="ws1", + fixture=fixture, + ) + + assert result.turn_results[0].skill_routing is False + + def test_run_agentic_conversation_deletes_metrics_even_when_a_later_turn_raises(): mock_client = MagicMock() mock_client.create_conversation.return_value = "conv-1"