|
| 1 | +import json |
| 2 | +import math |
1 | 3 |
|
2 | 4 | from .dirty_json import DirtyJson |
3 | 5 | import regex, re |
@@ -246,3 +248,241 @@ def replace_unescaped_newlines(match): |
246 | 248 | r'(?<=: ")(.*?)(?=")', replace_unescaped_newlines, json_string, flags=re.DOTALL |
247 | 249 | ) |
248 | 250 | return fixed_string |
| 251 | + |
| 252 | + |
| 253 | +_TOOL_XML_MARKER = re.compile( |
| 254 | + r"</?(?:tool_call\b|function(?:=|\b)|parameter(?:=|\b)|function_calls\b|" |
| 255 | + r"invoke\b|minimax:tool_call\b)|<\|tool_calls_section_begin\|>" |
| 256 | +) |
| 257 | +_TOOL_NAME = r"[A-Za-z0-9_.:-]+" |
| 258 | +_TOOL_FUNCTION = re.compile(rf"<function=({_TOOL_NAME})>") |
| 259 | +_TOOL_PARAMETER = re.compile(rf"<parameter=({_TOOL_NAME})>") |
| 260 | +_TOOL_CANONICAL_MARKER = re.compile(r"""(?:["'`]?\btool_(?:name|args)["'`]?)\s*:""") |
| 261 | +_TOOL_CODE = re.compile( |
| 262 | + r"(?m)^ {0,3}(`{3,}|~{3,})[^\n]*\n.*?(?:^ {0,3}\1[ \t]*(?:\n|$)|\Z)" |
| 263 | + r"|(`+)[^`]*?\2", |
| 264 | + re.DOTALL, |
| 265 | +) |
| 266 | + |
| 267 | + |
| 268 | +def _unique_json_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: |
| 269 | + result: dict[str, Any] = {} |
| 270 | + for key, value in pairs: |
| 271 | + if key in result: |
| 272 | + raise ValueError("duplicate_json_key") |
| 273 | + result[key] = value |
| 274 | + return result |
| 275 | + |
| 276 | + |
| 277 | +def _reject_json_constant(value: str) -> Any: |
| 278 | + raise ValueError("nonfinite_json_value") |
| 279 | + |
| 280 | + |
| 281 | +def _finite_json_float(value: str) -> float: |
| 282 | + number = float(value) |
| 283 | + if not math.isfinite(number): |
| 284 | + raise ValueError("nonfinite_json_value") |
| 285 | + return number |
| 286 | + |
| 287 | + |
| 288 | +_TOOL_JSON_DECODER = json.JSONDecoder( |
| 289 | + object_pairs_hook=_unique_json_object, |
| 290 | + parse_constant=_reject_json_constant, |
| 291 | + parse_float=_finite_json_float, |
| 292 | +) |
| 293 | + |
| 294 | + |
| 295 | +def _has_tool_marker(text: str) -> bool: |
| 296 | + return bool( |
| 297 | + _TOOL_XML_MARKER.search(text) |
| 298 | + or _TOOL_CANONICAL_MARKER.search(text) |
| 299 | + or ( |
| 300 | + re.search(r'"name"\s*:', text) |
| 301 | + and re.search(r'"(?:arguments|parameters)"\s*:', text) |
| 302 | + ) |
| 303 | + ) |
| 304 | + |
| 305 | + |
| 306 | +def _has_unclosed_json(text: str) -> bool: |
| 307 | + """Distinguish missing closing delimiters from complete invalid JSON.""" |
| 308 | + stack: list[str] = [] |
| 309 | + quoted = escaped = False |
| 310 | + for char in text: |
| 311 | + if quoted: |
| 312 | + if escaped: |
| 313 | + escaped = False |
| 314 | + elif char == "\\": |
| 315 | + escaped = True |
| 316 | + elif char == '"': |
| 317 | + quoted = False |
| 318 | + elif char == '"': |
| 319 | + quoted = True |
| 320 | + elif char in "{[": |
| 321 | + stack.append(char) |
| 322 | + elif char in "}]": |
| 323 | + if not stack or stack.pop() != {"}": "{", "]": "["}[char]: |
| 324 | + return False |
| 325 | + return bool(stack or quoted) |
| 326 | + |
| 327 | + |
| 328 | +def _strict_json_at_start(text: str) -> tuple[Any, int]: |
| 329 | + try: |
| 330 | + return _TOOL_JSON_DECODER.raw_decode(text) |
| 331 | + except json.JSONDecodeError as error: |
| 332 | + if _has_unclosed_json(text): |
| 333 | + raise EOFError("incomplete_json") from error |
| 334 | + raise ValueError("invalid_json") from error |
| 335 | + |
| 336 | + |
| 337 | +def _coerce_tool_parameter(value: str) -> Any: |
| 338 | + # Preserve file bodies, padding, and prose exactly. Never repair a value. |
| 339 | + if value and not any(char.isspace() for char in value): |
| 340 | + try: |
| 341 | + parsed, end = _TOOL_JSON_DECODER.raw_decode(value) |
| 342 | + if end == len(value): |
| 343 | + return parsed |
| 344 | + except (ValueError, RecursionError): |
| 345 | + pass |
| 346 | + return value |
| 347 | + |
| 348 | + |
| 349 | +def _leaked_function_at_start(text: str) -> tuple[dict[str, Any], int]: |
| 350 | + opening = _TOOL_FUNCTION.match(text) |
| 351 | + if opening is None: |
| 352 | + if ">" not in text: |
| 353 | + raise EOFError("incomplete_function_header") |
| 354 | + raise ValueError("invalid_function_header") |
| 355 | + end = text.find("</function>", opening.end()) |
| 356 | + if end < 0: |
| 357 | + raise EOFError("incomplete_function") |
| 358 | + body = text[opening.end() : end] |
| 359 | + args: dict[str, Any] = {} |
| 360 | + position = 0 |
| 361 | + while position < len(body): |
| 362 | + while position < len(body) and body[position].isspace(): |
| 363 | + position += 1 |
| 364 | + if position == len(body): |
| 365 | + break |
| 366 | + parameter = _TOOL_PARAMETER.match(body, position) |
| 367 | + if parameter is None: |
| 368 | + raise ValueError("invalid_parameter_markup") |
| 369 | + close = body.find("</parameter>", parameter.end()) |
| 370 | + if close < 0: |
| 371 | + raise ValueError("unclosed_parameter") |
| 372 | + key = parameter.group(1) |
| 373 | + value = body[parameter.end() : close] |
| 374 | + if key in args or _TOOL_XML_MARKER.search(value): |
| 375 | + raise ValueError("ambiguous_parameter") |
| 376 | + args[key] = _coerce_tool_parameter(value) |
| 377 | + position = close + len("</parameter>") |
| 378 | + return {"tool_name": opening.group(1), "tool_args": args}, end + len("</function>") |
| 379 | + |
| 380 | + |
| 381 | +def _leaked_payload_at_start(text: str) -> tuple[Any, int]: |
| 382 | + if text.startswith("<function="): |
| 383 | + return _leaked_function_at_start(text) |
| 384 | + if text.startswith(("{", "[")): |
| 385 | + return _strict_json_at_start(text) |
| 386 | + raise ValueError("unsupported_tool_syntax") |
| 387 | + |
| 388 | + |
| 389 | +def _leaked_call_candidate(text: str) -> Any: |
| 390 | + if text.startswith("<tool_call>"): |
| 391 | + inner = text[len("<tool_call>") :].lstrip() |
| 392 | + if not inner: |
| 393 | + raise EOFError("incomplete_tool_call") |
| 394 | + value, end = _leaked_payload_at_start(inner) |
| 395 | + tail = inner[end:].lstrip() |
| 396 | + if not tail.startswith("</tool_call>"): |
| 397 | + if "</tool_call>".startswith(tail): |
| 398 | + raise EOFError("incomplete_tool_call") |
| 399 | + raise ValueError("invalid_tool_call_wrapper") |
| 400 | + tail = tail[len("</tool_call>") :] |
| 401 | + else: |
| 402 | + value, end = _leaked_payload_at_start(text) |
| 403 | + tail = text[end:] |
| 404 | + if tail.strip(): |
| 405 | + raise ValueError("multiple_calls_or_surrounding_text") |
| 406 | + if isinstance(value, list): |
| 407 | + if len(value) != 1: |
| 408 | + raise ValueError("multiple_or_empty_calls") |
| 409 | + value = value[0] |
| 410 | + return value |
| 411 | + |
| 412 | + |
| 413 | +def _normalize_leaked_call(value: Any) -> dict[str, Any]: |
| 414 | + if not isinstance(value, dict): |
| 415 | + raise ValueError("invalid_call_object") |
| 416 | + if "tool_name" in value: |
| 417 | + if value.keys() - {"tool_name", "tool_args", "thoughts", "headline"}: |
| 418 | + raise ValueError("ambiguous_call_fields") |
| 419 | + if not isinstance(value.get("tool_args"), dict): |
| 420 | + raise ValueError("arguments_must_be_object") |
| 421 | + return {"tool_name": value["tool_name"], "tool_args": value["tool_args"]} |
| 422 | + if value.keys() - { |
| 423 | + "name", |
| 424 | + "arguments", |
| 425 | + "parameters", |
| 426 | + "type", |
| 427 | + "id", |
| 428 | + "thoughts", |
| 429 | + "headline", |
| 430 | + }: |
| 431 | + raise ValueError("ambiguous_call_fields") |
| 432 | + if "arguments" in value and "parameters" in value: |
| 433 | + raise ValueError("ambiguous_arguments") |
| 434 | + args = value.get("arguments", value.get("parameters")) |
| 435 | + if isinstance(args, str): |
| 436 | + encoded_args = args.strip() |
| 437 | + args, end = _strict_json_at_start(encoded_args) |
| 438 | + if end != len(encoded_args): |
| 439 | + raise ValueError("invalid_arguments_json") |
| 440 | + if not isinstance(args, dict): |
| 441 | + raise ValueError("arguments_must_be_object") |
| 442 | + if value.get("type", "function") != "function": |
| 443 | + raise ValueError("unsupported_call_type") |
| 444 | + return {"tool_name": value.get("name"), "tool_args": args} |
| 445 | + |
| 446 | + |
| 447 | +def extract_leaked_tool_request(response: str) -> tuple[str, dict[str, Any] | None]: |
| 448 | + """Classify completed text; the result hook authorizes any recovered request. |
| 449 | +
|
| 450 | + Return clean, quoted, call, truncated or rejected. The streaming/canonical |
| 451 | + parser retains its existing behavior; recovery never repairs argument values. |
| 452 | + """ |
| 453 | + if not isinstance(response, str) or not response.strip(): |
| 454 | + return "clean", None |
| 455 | + text = response.strip() |
| 456 | + if extract_tool_request(text) is not None: |
| 457 | + return "clean", None |
| 458 | + if text == "<tool_call": |
| 459 | + return "truncated", None |
| 460 | + |
| 461 | + # Parse the outer envelope before masking code inside argument strings. |
| 462 | + standalone = text.startswith(("<tool_call>", "<function=", "{", "[")) |
| 463 | + visible = text if standalone else _TOOL_CODE.sub("", text) |
| 464 | + if not _has_tool_marker(visible): |
| 465 | + return ("quoted" if _has_tool_marker(text) else "clean"), None |
| 466 | + if not standalone: |
| 467 | + return "rejected", None |
| 468 | + |
| 469 | + try: |
| 470 | + candidate = _leaked_call_candidate(text) |
| 471 | + if ( |
| 472 | + text.startswith("{") |
| 473 | + and isinstance(candidate, dict) |
| 474 | + and not ( |
| 475 | + {"tool_name", "tool_args", "actions"} & candidate.keys() |
| 476 | + or ("name" in candidate and {"arguments", "parameters"} & candidate.keys()) |
| 477 | + ) |
| 478 | + ): |
| 479 | + return "clean", None |
| 480 | + request = _normalize_leaked_call(candidate) |
| 481 | + name = request["tool_name"] |
| 482 | + if not isinstance(name, str) or not name or name != name.strip(): |
| 483 | + return "rejected", None |
| 484 | + return "call", request |
| 485 | + except EOFError: |
| 486 | + return "truncated", None |
| 487 | + except (ValueError, RecursionError): |
| 488 | + return "rejected", None |
0 commit comments