diff --git a/backend/open_webui/utils/files.py b/backend/open_webui/utils/files.py index 3bb918e8d..06bec3325 100644 --- a/backend/open_webui/utils/files.py +++ b/backend/open_webui/utils/files.py @@ -148,7 +148,7 @@ def get_audio_url_from_base64(request, base64_audio_string, metadata, user): def get_file_url_from_base64(request, base64_file_string, metadata, user): - if 'data:image/png;base64' in base64_file_string: + if BASE64_IMAGE_URL_PREFIX.match(base64_file_string): return get_image_url_from_base64(request, base64_file_string, metadata, user) elif 'data:audio/wav;base64' in base64_file_string: return get_audio_url_from_base64(request, base64_file_string, metadata, user) diff --git a/backend/open_webui/utils/middleware.py b/backend/open_webui/utils/middleware.py index cbf644b53..058c8db16 100644 --- a/backend/open_webui/utils/middleware.py +++ b/backend/open_webui/utils/middleware.py @@ -1049,6 +1049,13 @@ def process_tool_result( tool_result_files = [] + # Detect base64 image data URIs from tool results (e.g. binary image + # responses from execute_tool_server). Move the data URI to + # tool_result_files and replace tool_result with a text summary. + if isinstance(tool_result, str) and tool_result.startswith('data:image/'): + tool_result_files.append({'type': 'image', 'url': tool_result}) + tool_result = f'{tool_function_name}: Image file read successfully.' + if isinstance(tool_result, list): if tool_type == 'mcp': # MCP tool_response = [] @@ -4181,19 +4188,27 @@ async def streaming_chat_response_handler(response, ctx): break for result in results: + output_parts = [{'type': 'input_text', 'text': result.get('content', '')}] + + # Separate image data URIs (for LLM via input_image) from + # other files (for frontend display via files attribute). + display_files = [] + for file_item in result.get('files', []): + if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'): + # LLM-only: add as input_image part (invisible to serialize_output) + output_parts.append({'type': 'input_image', 'image_url': file_item['url']}) + else: + # Frontend display (MCP images, audio, etc.) + display_files.append(file_item) + output.append( { 'type': 'function_call_output', 'id': output_id('fco'), 'call_id': result.get('tool_call_id', ''), - 'output': [ - { - 'type': 'input_text', - 'text': result.get('content', ''), - } - ], + 'output': output_parts, 'status': 'completed', - **({'files': result.get('files')} if result.get('files') else {}), + **({'files': display_files} if display_files else {}), **({'embeds': result.get('embeds')} if result.get('embeds') else {}), } ) @@ -4262,12 +4277,23 @@ async def streaming_chat_response_handler(response, ctx): ) tool_call_sources.clear() + # Strip input_image parts (large base64 data URIs) from the + # output sent to the frontend — they're only for LLM consumption + # via convert_output_to_messages. + frontend_output = [] + for item in output: + if item.get('type') == 'function_call_output': + parts = item.get('output', []) + if any(p.get('type') == 'input_image' for p in parts): + item = {**item, 'output': [p for p in parts if p.get('type') != 'input_image']} + frontend_output.append(item) + await event_emitter( { 'type': 'chat:completion', 'data': { 'content': serialize_output(output), - 'output': output, + 'output': frontend_output, }, } ) @@ -4287,11 +4313,35 @@ async def streaming_chat_response_handler(response, ctx): ) new_form_data['previous_response_id'] = last_response_id else: + tool_messages = convert_output_to_messages(output, raw=True) + + # Chat Completions providers don't support multimodal + # tool messages. Extract images into a user message. + image_urls = [] + for message in tool_messages: + if message.get('role') == 'tool' and isinstance(message.get('content'), list): + text_parts = [] + for part in message['content']: + if part.get('type') == 'input_text': + text_parts.append(part.get('text', '')) + elif part.get('type') == 'input_image': + image_urls.append(part.get('image_url', '')) + message['content'] = ''.join(text_parts) + new_form_data['messages'] = [ *form_data['messages'], - *convert_output_to_messages(output, raw=True), + *tool_messages, ] + if image_urls: + new_form_data['messages'].append({ + 'role': 'user', + 'content': [ + {'type': 'text', 'text': 'Here are the images from the tool results above. Please analyze them.'}, + *[{'type': 'image_url', 'image_url': {'url': url}} for url in image_urls], + ], + }) + res = await generate_chat_completion( request, new_form_data, @@ -4416,7 +4466,7 @@ async def streaming_chat_response_handler(response, ctx): if isinstance(stdout, str): stdoutLines = stdout.split('\n') for idx, line in enumerate(stdoutLines): - if 'data:image/png;base64' in line: + if re.match(r'data:image/\w+;base64', line): image_url = get_image_url_from_base64( request, line, @@ -4433,7 +4483,7 @@ async def streaming_chat_response_handler(response, ctx): if isinstance(result, str): resultLines = result.split('\n') for idx, line in enumerate(resultLines): - if 'data:image/png;base64' in line: + if re.match(r'data:image/\w+;base64', line): image_url = get_image_url_from_base64( request, line, diff --git a/backend/open_webui/utils/misc.py b/backend/open_webui/utils/misc.py index 91217b3c2..060d5c4a6 100644 --- a/backend/open_webui/utils/misc.py +++ b/backend/open_webui/utils/misc.py @@ -196,21 +196,39 @@ def convert_output_to_messages(output: list, raw: bool = False) -> list[dict]: # Flush any pending content/tool_calls before adding tool result flush_pending() - # Extract text from output content parts + # Extract text and images from output content parts output_parts = item.get('output', []) content = '' + image_urls = [] for part in output_parts: if part.get('type') == 'input_text': output_text = part.get('text', '') content += str(output_text) if not isinstance(output_text, str) else output_text + elif part.get('type') == 'input_image': + url = part.get('image_url', '') + if url: + image_urls.append(url) - messages.append( - { - 'role': 'tool', - 'tool_call_id': item.get('call_id', ''), - 'content': content, - } - ) + if image_urls: + # Multimodal tool content with image(s) + messages.append( + { + 'role': 'tool', + 'tool_call_id': item.get('call_id', ''), + 'content': [ + {'type': 'input_text', 'text': content}, + *[{'type': 'input_image', 'image_url': url} for url in image_urls], + ], + } + ) + else: + messages.append( + { + 'role': 'tool', + 'tool_call_id': item.get('call_id', ''), + 'content': content, + } + ) elif item_type == 'reasoning': if raw: diff --git a/backend/open_webui/utils/tools.py b/backend/open_webui/utils/tools.py index 5d88facce..e95942f94 100644 --- a/backend/open_webui/utils/tools.py +++ b/backend/open_webui/utils/tools.py @@ -1,3 +1,4 @@ +import base64 import inspect import logging import re @@ -1236,9 +1237,13 @@ async def execute_tool_server( if param_name in params: if param_in == 'path': path_params[param_name] = params[param_name] - elif param_in == 'query': - if params[param_name] is not None: - query_params[param_name] = params[param_name] + if param_in == 'query': + value = params[param_name] + # Skip empty values for optional params (LLMs sometimes + # pass "" instead of omitting optional parameters). + if value is None or (value == '' and not param.get('required')): + continue + query_params[param_name] = value final_url = f'{url.rstrip("/")}{route_path}' for key, value in path_params.items(): @@ -1273,7 +1278,13 @@ async def execute_tool_server( try: response_data = await response.json() except Exception: - response_data = await response.text() + content_type = response.headers.get('Content-Type', '').split(';')[0].strip() + if content_type.startswith('text/') or not content_type: + response_data = await response.text() + else: + raw = await response.read() + b64 = base64.b64encode(raw).decode() + response_data = f'data:{content_type};base64,{b64}' response_headers = response.headers return (response_data, response_headers) @@ -1292,7 +1303,13 @@ async def execute_tool_server( try: response_data = await response.json() except Exception: - response_data = await response.text() + content_type = response.headers.get('Content-Type', '').split(';')[0].strip() + if content_type.startswith('text/') or not content_type: + response_data = await response.text() + else: + raw = await response.read() + b64 = base64.b64encode(raw).decode() + response_data = f'data:{content_type};base64,{b64}' response_headers = response.headers return (response_data, response_headers)