Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/fix-tool-approval-sequence.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'ai': patch
---

fix: reject trailing messages after tool approval responses
62 changes: 62 additions & 0 deletions packages/ai/src/generate-text/stream-text.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import {
import { mockSandboxSessionFileStubs } from '../test/mock-sandbox';
import { z } from 'zod/v4';
import { Output, type LanguageModelCallEndEvent, type Telemetry } from '..';
import { MissingToolResultsError } from '../error/missing-tool-result-error';
import * as logWarningsModule from '../logger/log-warnings';
import type { Instructions } from '../prompt';
import { MockLanguageModelV4 } from '../test/mock-language-model-v4';
Expand Down Expand Up @@ -27061,6 +27062,67 @@ describe('streamText', () => {
});
});

it('should reject a user message after a tool approval response', async () => {
const executeFunction = vi.fn().mockReturnValue('result1');
const doStream = vi.fn(async () => {
throw new Error('model should not be called');
});
const onError = vi.fn();

const result = streamText({
model: new MockLanguageModelV4({ doStream }),
tools: {
tool1: tool({
inputSchema: z.object({ value: z.string() }),
execute: executeFunction,
}),
},
toolApproval: {
tool1: 'user-approval',
},
onError,
messages: [
{ role: 'user', content: 'test-input' },
{
role: 'assistant',
content: [
{
input: { value: 'value' },
toolCallId: 'call-1',
toolName: 'tool1',
type: 'tool-call',
},
{
approvalId: 'id-1',
toolCallId: 'call-1',
type: 'tool-approval-request',
},
],
},
{
role: 'tool',
content: [
{
approvalId: 'id-1',
type: 'tool-approval-response',
approved: true,
},
],
},
{ role: 'user', content: 'additional context' },
],
});

await result.consumeStream();

expect(onError).toHaveBeenCalledOnce();
const error = onError.mock.calls[0][0].error;
expect(MissingToolResultsError.isInstance(error)).toBe(true);
expect(error).toMatchObject({ toolCallIds: ['call-1'] });
expect(executeFunction).not.toHaveBeenCalled();
expect(doStream).not.toHaveBeenCalled();
});

describe('when a call from a single tool that needs approval is approved', () => {
let result: StreamTextResult<any, any, any>;
let prompts: LanguageModelV4Prompt[];
Expand Down
28 changes: 12 additions & 16 deletions packages/ai/src/prompt/convert-to-language-model-prompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,15 +69,16 @@ export async function convertToLanguageModelPrompt({
}
}

const approvedToolCallIds = new Set<string>();
for (const message of prompt.messages) {
if (message.role === 'tool') {
for (const part of message.content) {
if (part.type === 'tool-approval-response') {
const toolCallId = approvalIdToToolCallId.get(part.approvalId);
if (toolCallId) {
approvedToolCallIds.add(toolCallId);
}
// collectToolApprovals only consumes responses from the final tool message.
// Older responses must not suppress missing-result validation.
const approvalResponseToolCallIds = new Set<string>();
const lastMessage = prompt.messages.at(-1);
if (lastMessage?.role === 'tool') {
for (const part of lastMessage.content) {
if (part.type === 'tool-approval-response') {
const toolCallId = approvalIdToToolCallId.get(part.approvalId);
if (toolCallId) {
approvalResponseToolCallIds.add(toolCallId);
}
}
}
Expand Down Expand Up @@ -136,11 +137,6 @@ export async function convertToLanguageModelPrompt({
}
case 'user':
case 'system':
// remove approved tool calls from the set before checking:
for (const id of approvedToolCallIds) {
toolCallIds.delete(id);
}

if (toolCallIds.size > 0) {
throw new MissingToolResultsError({
toolCallIds: Array.from(toolCallIds),
Expand All @@ -150,8 +146,8 @@ export async function convertToLanguageModelPrompt({
}
}

// remove approved tool calls from the set before checking:
for (const id of approvedToolCallIds) {
// The final approval response is consumed before the next model call.
for (const id of approvalResponseToolCallIds) {
toolCallIds.delete(id);
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest';
import type { ModelMessage } from '@ai-sdk/provider-utils';
import { convertToLanguageModelPrompt } from './convert-to-language-model-prompt';
import { MissingToolResultsError } from '../error/missing-tool-result-error';

Expand Down Expand Up @@ -71,6 +72,138 @@ describe('tool validation', () => {
expect(result).toMatchSnapshot();
});

it.each<{
decision: string;
approved: boolean;
name: string;
trailingMessage: ModelMessage;
}>([
{
decision: 'approved',
approved: true,
name: 'user',
trailingMessage: { role: 'user', content: 'additional context' },
},
{
decision: 'approved',
approved: true,
name: 'system',
trailingMessage: { role: 'system', content: 'additional instructions' },
},
{
decision: 'approved',
approved: true,
name: 'assistant',
trailingMessage: { role: 'assistant', content: 'continued response' },
},
{
decision: 'denied',
approved: false,
name: 'user',
trailingMessage: { role: 'user', content: 'additional context' },
},
])(
'should reject a $name message after an $decision tool approval response',
async ({ approved, trailingMessage }) => {
await expect(
convertToLanguageModelPrompt({
prompt: {
instructions: undefined,
messages: [
{
role: 'assistant',
content: [
{
type: 'tool-call',
toolCallId: 'call_to_approve',
toolName: 'dangerous_action',
input: { action: 'delete_db' },
},
{
type: 'tool-approval-request',
toolCallId: 'call_to_approve',
approvalId: 'approval_123',
},
],
},
{
role: 'tool',
content: [
{
type: 'tool-approval-response',
approvalId: 'approval_123',
approved,
},
],
},
trailingMessage,
],
},
supportedUrls: {},
download: undefined,
}),
).rejects.toEqual(
expect.objectContaining({
name: 'AI_MissingToolResultsError',
toolCallIds: ['call_to_approve'],
}),
);
},
);

it('should allow later messages after an approved tool call has a result', async () => {
const result = await convertToLanguageModelPrompt({
prompt: {
instructions: undefined,
messages: [
{
role: 'assistant',
content: [
{
type: 'tool-call',
toolCallId: 'call_to_approve',
toolName: 'dangerous_action',
input: { action: 'delete_db' },
},
{
type: 'tool-approval-request',
toolCallId: 'call_to_approve',
approvalId: 'approval_123',
},
],
},
{
role: 'tool',
content: [
{
type: 'tool-approval-response',
approvalId: 'approval_123',
approved: true,
},
{
type: 'tool-result',
toolCallId: 'call_to_approve',
toolName: 'dangerous_action',
output: { type: 'text', value: 'completed' },
},
],
},
{ role: 'assistant', content: 'The action completed.' },
{ role: 'user', content: 'What happened?' },
],
},
supportedUrls: {},
download: undefined,
});

expect(result.map(message => message.role)).toEqual([
'assistant',
'tool',
'assistant',
'user',
]);
});

it('should preserve provider-executed tool-approval-response', async () => {
const result = await convertToLanguageModelPrompt({
prompt: {
Expand Down