From d52da6456d4fcfcd8da0266e7eca60e56f016012 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:35:04 +0000 Subject: [PATCH 1/3] Initial plan From 1c75b600129f6f0b95bd5e65966c6b786dcbf81d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:44:38 +0000 Subject: [PATCH 2/3] Fix duplicate helpers: import _get_tool_calls_results and _reformat_tool_calls_results from _common/utils.py - Add _stringify_tool_result and _log_safe_summary to _common/utils.py - Add improved _get_tool_calls_results and _reformat_tool_calls_results to _common/utils.py (using _format_value and _stringify_tool_result for consistent arg/result formatting) - Remove duplicate local definitions from _tool_call_success.py - Import _get_tool_calls_results and _reformat_tool_calls_results from _common/utils.py in _tool_call_success.py Co-authored-by: m7md7sien <16615690+m7md7sien@users.noreply.github.com> --- .../azure/ai/evaluation/_common/utils.py | 137 ++++++++++++++++++ .../_tool_call_success/_tool_call_success.py | 65 +-------- 2 files changed, 138 insertions(+), 64 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py index d0e7f8e0f77e..f7185fffeed9 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py @@ -1,6 +1,7 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +import json import os import posixpath import re @@ -907,6 +908,142 @@ def simplify_messages(messages, drop_system=True, drop_tool_calls=False, logger= return messages +def _stringify_tool_result(result): + """Render a tool_result value as a string the LLM judge can read. + + Tool outputs arrive in mixed shapes depending on the producer: function/MCP tools usually + emit a plain ``str``, while built-in grounding tools (``azure_ai_search``, ``azure_fabric``, + ``sharepoint_grounding``) emit a list/dict. Falling back to ``f"{result}"`` for the latter + produced a Python ``repr`` (single quotes, trailing commas) that the LLM had to + reverse-engineer. Strings are passed through unchanged (zero behavior change for function/MCP + tools), anything else is serialized as JSON with ``default=str`` so non-JSON-native values do + not raise, and ``None`` renders as the empty string. + + :param result: The raw tool_result value. + :type result: Any + :return: A string representation suitable for an LLM prompt. + :rtype: str + """ + if result is None: + return "" + if isinstance(result, str): + return result + try: + return json.dumps(result, default=str, ensure_ascii=False) + except (TypeError, ValueError): + return str(result) + + +def _log_safe_summary(obj): + """Return a non-sensitive structural summary of a payload for safe logging. + + The raw payload may contain customer-controlled data (tool arguments, tool results, assistant + text, database rows, file content, etc.) which can include credentials or PII. Logging the + payload itself risks leaking that data into telemetry sinks at any log level. This helper + returns shape-only metadata - type, length, top-level keys/roles - which is sufficient to + diagnose schema drift without exposing values. + + :param obj: The payload to summarize. + :type obj: Any + :return: A shape-only, non-sensitive summary string. + :rtype: str + """ + try: + type_name = type(obj).__name__ + if isinstance(obj, list): + roles = [] + for item in obj[:10]: + if isinstance(item, dict): + role = item.get("role") + if isinstance(role, str): + roles.append(role) + roles_summary = roles if roles else "n/a" + return f"type={type_name} len={len(obj)} roles={roles_summary}" + if isinstance(obj, dict): + keys = sorted(k for k in obj.keys() if isinstance(k, str))[:10] + return f"type={type_name} top_keys={keys}" + length = len(obj) if hasattr(obj, "__len__") else "n/a" + return f"type={type_name} len={length}" + except Exception: # pylint: disable=broad-except + return f"type={type(obj).__name__} (summary unavailable)" + + +def _get_tool_calls_results(agent_response_msgs): + """Extract formatted agent tool calls and results from a response. + + The output uses the ``[TOOL_CALL]`` / ``[TOOL_RESULT]`` line format. Tool results are rendered + via :func:`_stringify_tool_result` so list/dict grounding outputs are readable JSON. + + :param agent_response_msgs: The agent response messages to scan. + :type agent_response_msgs: List[dict] + :return: A list of formatted tool-call/result lines. + :rtype: List[str] + """ + agent_response_text = [] + tool_results = {} + + for msg in agent_response_msgs: + if msg.get("role") == "tool" and "tool_call_id" in msg: + for content in msg.get("content", []): + if content.get("type") == "tool_result": + result = content.get("tool_result") + tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {_stringify_tool_result(result)}" + + for msg in agent_response_msgs: + if "role" in msg and msg.get("role") == "assistant" and "content" in msg: + for content in msg.get("content", []): + if content.get("type") == "tool_call": + if "tool_call" in content and "function" in content.get("tool_call", {}): + tc = content.get("tool_call", {}) + func_name = tc.get("function", {}).get("name", "") + args = tc.get("function", {}).get("arguments", {}) + tool_call_id = tc.get("id") + else: + tool_call_id = content.get("tool_call_id") + func_name = content.get("name", "") + args = content.get("arguments", {}) + args_str = ", ".join(f"{k}={_format_value(v)}" for k, v in args.items()) + call_line = f"[TOOL_CALL] {func_name}({args_str})" + agent_response_text.append(call_line) + if tool_call_id in tool_results: + agent_response_text.append(tool_results[tool_call_id]) + + return agent_response_text + + +def _reformat_tool_calls_results(response, logger=None): + """Reformat an agent response into tool-call/result lines, with a safe fallback. + + :param response: The agent response to reformat. + :type response: Union[None, List[dict], str] + :param logger: Optional logger for warning messages. + :type logger: Optional[logging.Logger] + :return: The formatted string, or the original response if parsing fails. + :rtype: Union[str, List[dict]] + """ + try: + if response is None or response == []: + return "" + agent_response = _get_tool_calls_results(response) + if agent_response == []: + if logger: + logger.warning( + "Empty agent response extracted, likely due to input schema change. " + "Falling back to using the original response. %s", + _log_safe_summary(response), + ) + return response + return "\n".join(agent_response) + except Exception as e: # pylint: disable=broad-except + if logger: + logger.warning( + "Agent response could not be parsed, falling back to original response. Error: %s. %s", + e, + _log_safe_summary(response), + ) + return response + + def upload(path: str, container_client: ContainerClient, logger=None): """Upload files or directories to Azure Blob Storage using a container client. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py index 44e0876bad68..052fa0b41573 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py @@ -15,6 +15,7 @@ from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase from azure.ai.evaluation._evaluators._common._validators import ToolDefinitionsValidator, ValidatorInterface from azure.ai.evaluation._common._experimental import experimental +from azure.ai.evaluation._common.utils import _get_tool_calls_results, _reformat_tool_calls_results logger = logging.getLogger(__name__) @@ -271,70 +272,6 @@ def _filter_to_used_tools(tool_definitions, msgs_list, logger=None): return tool_definitions -def _get_tool_calls_results(agent_response_msgs): - """Extract formatted agent tool calls and results from response.""" - agent_response_text = [] - tool_results = {} - - # First pass: collect tool results - - for msg in agent_response_msgs: - if msg.get("role") == "tool" and "tool_call_id" in msg: - for content in msg.get("content", []): - if content.get("type") == "tool_result": - result = content.get("tool_result") - tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {result}" - - # Second pass: parse assistant messages and tool calls - for msg in agent_response_msgs: - if "role" in msg and msg.get("role") == "assistant" and "content" in msg: - - for content in msg.get("content", []): - - if content.get("type") == "tool_call": - if "tool_call" in content and "function" in content.get("tool_call", {}): - tc = content.get("tool_call", {}) - func_name = tc.get("function", {}).get("name", "") - args = tc.get("function", {}).get("arguments", {}) - tool_call_id = tc.get("id") - else: - tool_call_id = content.get("tool_call_id") - func_name = content.get("name", "") - args = content.get("arguments", {}) - args_str = ", ".join(f'{k}="{v}"' for k, v in args.items()) - call_line = f"[TOOL_CALL] {func_name}({args_str})" - agent_response_text.append(call_line) - if tool_call_id in tool_results: - agent_response_text.append(tool_results[tool_call_id]) - - return agent_response_text - - -def _reformat_tool_calls_results(response, logger=None): - try: - if response is None or response == []: - return "" - agent_response = _get_tool_calls_results(response) - if agent_response == []: - # If no message could be extracted, likely the format changed, - # fallback to the original response in that case - if logger: - logger.warning( - f"Empty agent response extracted, likely due to input schema change. " - f"Falling back to using the original response" - ) - return response - return "\n".join(agent_response) - except Exception: - # If the agent response cannot be parsed for whatever - # reason (e.g. the converter format changed), the original response is returned - # This is a fallback to ensure that the evaluation can still proceed. - # See comments on reformat_conversation_history for more details. - if logger: - logger.warning(f"Agent response could not be parsed, falling back to original response") - return response - - def _reformat_tool_definitions(tool_definitions, logger=None): try: output_lines = ["TOOL_DEFINITIONS:"] From 84adef0023ccd40eb0b966b7a4b50dd4eaced7dc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:59:35 +0000 Subject: [PATCH 3/3] Resolve merge conflicts with origin/main Co-authored-by: m7md7sien <16615690+m7md7sien@users.noreply.github.com> --- .../azure/ai/evaluation/_common/utils.py | 9 --- .../_tool_call_success/_tool_call_success.py | 74 +------------------ 2 files changed, 1 insertion(+), 82 deletions(-) diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py index a6084fef14ab..b54677cf1b13 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py @@ -930,13 +930,10 @@ def simplify_messages(messages, drop_system=True, drop_tool_calls=False, logger= return messages -<<<<<<< HEAD -======= # Runtime tool-call statuses that indicate a failed or incomplete execution. _FAILED_RUNTIME_STATUSES = frozenset({"failed", "incomplete"}) ->>>>>>> origin/main def _stringify_tool_result(result): """Render a tool_result value as a string the LLM judge can read. @@ -997,8 +994,6 @@ def _log_safe_summary(obj): return f"type={type(obj).__name__} (summary unavailable)" -<<<<<<< HEAD -======= def _coerce_bool(value) -> Optional[bool]: """Coerce an LLM output value to bool or None. @@ -1117,7 +1112,6 @@ def _collect_failed_tool_calls(messages): return ordered ->>>>>>> origin/main def _get_tool_calls_results(agent_response_msgs): """Extract formatted agent tool calls and results from a response. @@ -1194,8 +1188,6 @@ def _reformat_tool_calls_results(response, logger=None): return response -<<<<<<< HEAD -======= def _get_conversation_history_with_tool_calls(query, include_system_messages=False): """Parse conversation history, rendering tool calls/results inline within agent turns. @@ -1286,7 +1278,6 @@ def _get_conversation_history_with_tool_calls(query, include_system_messages=Fal return result ->>>>>>> origin/main def upload(path: str, container_client: ContainerClient, logger=None): """Upload files or directories to Azure Blob Storage using a container client. diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py index 1085b205ff73..ce45218af2f8 100644 --- a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py +++ b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py @@ -14,7 +14,7 @@ ) from azure.ai.evaluation._evaluators._common import PromptyEvaluatorBase from azure.ai.evaluation._evaluators._common._validators import ToolDefinitionsValidator, ValidatorInterface -from azure.ai.evaluation._common.utils import _log_safe_summary, _stringify_tool_result +from azure.ai.evaluation._common.utils import _log_safe_summary from azure.ai.evaluation._common._experimental import experimental from azure.ai.evaluation._common.utils import _get_tool_calls_results, _reformat_tool_calls_results @@ -273,78 +273,6 @@ def _filter_to_used_tools(tool_definitions, msgs_list, logger=None): return tool_definitions -<<<<<<< HEAD -======= -def _get_tool_calls_results(agent_response_msgs): - """Extract formatted agent tool calls and results from response.""" - agent_response_text = [] - tool_results = {} - - # First pass: collect tool results - - for msg in agent_response_msgs: - if msg.get("role") == "tool" and "tool_call_id" in msg: - for content in msg.get("content", []): - if content.get("type") == "tool_result": - result = content.get("tool_result") - tool_results[msg["tool_call_id"]] = f"[TOOL_RESULT] {_stringify_tool_result(result)}" - - # Second pass: parse assistant messages and tool calls - for msg in agent_response_msgs: - if "role" in msg and msg.get("role") == "assistant" and "content" in msg: - - for content in msg.get("content", []): - - if content.get("type") == "tool_call": - if "tool_call" in content and "function" in content.get("tool_call", {}): - tc = content.get("tool_call", {}) - func_name = tc.get("function", {}).get("name", "") - args = tc.get("function", {}).get("arguments", {}) - tool_call_id = tc.get("id") - else: - tool_call_id = content.get("tool_call_id") - func_name = content.get("name", "") - args = content.get("arguments", {}) - args_str = ", ".join(f'{k}="{v}"' for k, v in args.items()) - call_line = f"[TOOL_CALL] {func_name}({args_str})" - agent_response_text.append(call_line) - if tool_call_id in tool_results: - agent_response_text.append(tool_results[tool_call_id]) - - return agent_response_text - - -def _reformat_tool_calls_results(response, logger=None): - try: - if response is None or response == []: - return "" - agent_response = _get_tool_calls_results(response) - if agent_response == []: - # If no message could be extracted, likely the format changed, - # fallback to the original response in that case - if logger: - logger.warning( - "Empty agent response extracted, likely due to input schema change. " - "Falling back to using the original response. %s", - _log_safe_summary(response), - ) - return response - return "\n".join(agent_response) - except Exception as e: # pylint: disable=broad-except - # If the agent response cannot be parsed for whatever - # reason (e.g. the converter format changed), the original response is returned - # This is a fallback to ensure that the evaluation can still proceed. - # See comments on reformat_conversation_history for more details. - if logger: - logger.warning( - "Agent response could not be parsed, falling back to original response. Error: %s. %s", - e, - _log_safe_summary(response), - ) - return response - - ->>>>>>> origin/main def _reformat_tool_definitions(tool_definitions, logger=None): try: output_lines = ["TOOL_DEFINITIONS:"]