Emil_Shanaty/open-webui-ai4me
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refac

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-03-23 19:46:24 -05:00
parent 36c3fc58b5
commit 1c25b06dca
4 changed files with 110 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/utils/files.py
+1 -1
View File
@@ -148,7 +148,7 @@ def get_audio_url_from_base64(request, base64_audio_string, metadata, user):
def get_file_url_from_base64(request, base64_file_string, metadata, user): def get_file_url_from_base64(request, base64_file_string, metadata, user):
if 'data:image/png;base64' in base64_file_string: if BASE64_IMAGE_URL_PREFIX.match(base64_file_string):
return get_image_url_from_base64(request, base64_file_string, metadata, user) return get_image_url_from_base64(request, base64_file_string, metadata, user)
elif 'data:audio/wav;base64' in base64_file_string: elif 'data:audio/wav;base64' in base64_file_string:
return get_audio_url_from_base64(request, base64_file_string, metadata, user) return get_audio_url_from_base64(request, base64_file_string, metadata, user)
backend/open_webui/utils/middleware.py
+61 -11
View File
@@ -1049,6 +1049,13 @@ def process_tool_result(
tool_result_files = [] tool_result_files = []
# Detect base64 image data URIs from tool results (e.g. binary image
# responses from execute_tool_server). Move the data URI to
# tool_result_files and replace tool_result with a text summary.
if isinstance(tool_result, str) and tool_result.startswith('data:image/'):
tool_result_files.append({'type': 'image', 'url': tool_result})
tool_result = f'{tool_function_name}: Image file read successfully.'
if isinstance(tool_result, list): if isinstance(tool_result, list):
if tool_type == 'mcp': # MCP if tool_type == 'mcp': # MCP
tool_response = [] tool_response = []
@@ -4181,19 +4188,27 @@ async def streaming_chat_response_handler(response, ctx):
break break
for result in results: for result in results:
output_parts = [{'type': 'input_text', 'text': result.get('content', '')}]
# Separate image data URIs (for LLM via input_image) from
# other files (for frontend display via files attribute).
display_files = []
for file_item in result.get('files', []):
if file_item.get('type') == 'image' and file_item.get('url', '').startswith('data:'):
# LLM-only: add as input_image part (invisible to serialize_output)
output_parts.append({'type': 'input_image', 'image_url': file_item['url']})
else:
# Frontend display (MCP images, audio, etc.)
display_files.append(file_item)
output.append( output.append(
{ {
'type': 'function_call_output', 'type': 'function_call_output',
'id': output_id('fco'), 'id': output_id('fco'),
'call_id': result.get('tool_call_id', ''), 'call_id': result.get('tool_call_id', ''),
'output': [ 'output': output_parts,
{
'type': 'input_text',
'text': result.get('content', ''),
}
],
'status': 'completed', 'status': 'completed',
**({'files': result.get('files')} if result.get('files') else {}), **({'files': display_files} if display_files else {}),
**({'embeds': result.get('embeds')} if result.get('embeds') else {}), **({'embeds': result.get('embeds')} if result.get('embeds') else {}),
} }
) )
@@ -4262,12 +4277,23 @@ async def streaming_chat_response_handler(response, ctx):
) )
tool_call_sources.clear() tool_call_sources.clear()
# Strip input_image parts (large base64 data URIs) from the
# output sent to the frontend — they're only for LLM consumption
# via convert_output_to_messages.
frontend_output = []
for item in output:
if item.get('type') == 'function_call_output':
parts = item.get('output', [])
if any(p.get('type') == 'input_image' for p in parts):
item = {**item, 'output': [p for p in parts if p.get('type') != 'input_image']}
frontend_output.append(item)
await event_emitter( await event_emitter(
{ {
'type': 'chat:completion', 'type': 'chat:completion',
'data': { 'data': {
'content': serialize_output(output), 'content': serialize_output(output),
'output': output, 'output': frontend_output,
}, },
} }
) )
@@ -4287,11 +4313,35 @@ async def streaming_chat_response_handler(response, ctx):
) )
new_form_data['previous_response_id'] = last_response_id new_form_data['previous_response_id'] = last_response_id
else: else:
tool_messages = convert_output_to_messages(output, raw=True)
# Chat Completions providers don't support multimodal
# tool messages. Extract images into a user message.
image_urls = []
for message in tool_messages:
if message.get('role') == 'tool' and isinstance(message.get('content'), list):
text_parts = []
for part in message['content']:
if part.get('type') == 'input_text':
text_parts.append(part.get('text', ''))
elif part.get('type') == 'input_image':
image_urls.append(part.get('image_url', ''))
message['content'] = ''.join(text_parts)
new_form_data['messages'] = [ new_form_data['messages'] = [
*form_data['messages'], *form_data['messages'],
*convert_output_to_messages(output, raw=True), *tool_messages,
] ]
if image_urls:
new_form_data['messages'].append({
'role': 'user',
'content': [
{'type': 'text', 'text': 'Here are the images from the tool results above. Please analyze them.'},
*[{'type': 'image_url', 'image_url': {'url': url}} for url in image_urls],
],
})
res = await generate_chat_completion( res = await generate_chat_completion(
request, request,
new_form_data, new_form_data,
@@ -4416,7 +4466,7 @@ async def streaming_chat_response_handler(response, ctx):
if isinstance(stdout, str): if isinstance(stdout, str):
stdoutLines = stdout.split('\n') stdoutLines = stdout.split('\n')
for idx, line in enumerate(stdoutLines): for idx, line in enumerate(stdoutLines):
if 'data:image/png;base64' in line: if re.match(r'data:image/\w+;base64', line):
image_url = get_image_url_from_base64( image_url = get_image_url_from_base64(
request, request,
line, line,
@@ -4433,7 +4483,7 @@ async def streaming_chat_response_handler(response, ctx):
if isinstance(result, str): if isinstance(result, str):
resultLines = result.split('\n') resultLines = result.split('\n')
for idx, line in enumerate(resultLines): for idx, line in enumerate(resultLines):
if 'data:image/png;base64' in line: if re.match(r'data:image/\w+;base64', line):
image_url = get_image_url_from_base64( image_url = get_image_url_from_base64(
request, request,
line, line,
backend/open_webui/utils/misc.py
+26 -8
View File
@@ -196,21 +196,39 @@ def convert_output_to_messages(output: list, raw: bool = False) -> list[dict]:
# Flush any pending content/tool_calls before adding tool result # Flush any pending content/tool_calls before adding tool result
flush_pending() flush_pending()
# Extract text from output content parts # Extract text and images from output content parts
output_parts = item.get('output', []) output_parts = item.get('output', [])
content = '' content = ''
image_urls = []
for part in output_parts: for part in output_parts:
if part.get('type') == 'input_text': if part.get('type') == 'input_text':
output_text = part.get('text', '') output_text = part.get('text', '')
content += str(output_text) if not isinstance(output_text, str) else output_text content += str(output_text) if not isinstance(output_text, str) else output_text
elif part.get('type') == 'input_image':
url = part.get('image_url', '')
if url:
image_urls.append(url)
messages.append( if image_urls:
{ # Multimodal tool content with image(s)
'role': 'tool', messages.append(
'tool_call_id': item.get('call_id', ''), {
'content': content, 'role': 'tool',
} 'tool_call_id': item.get('call_id', ''),
) 'content': [
{'type': 'input_text', 'text': content},
*[{'type': 'input_image', 'image_url': url} for url in image_urls],
],
}
)
else:
messages.append(
{
'role': 'tool',
'tool_call_id': item.get('call_id', ''),
'content': content,
}
)
elif item_type == 'reasoning': elif item_type == 'reasoning':
if raw: if raw:
backend/open_webui/utils/tools.py
+22 -5
View File
@@ -1,3 +1,4 @@
import base64
import inspect import inspect
import logging import logging
import re import re
@@ -1236,9 +1237,13 @@ async def execute_tool_server(
if param_name in params: if param_name in params:
if param_in == 'path': if param_in == 'path':
path_params[param_name] = params[param_name] path_params[param_name] = params[param_name]
elif param_in == 'query': if param_in == 'query':
if params[param_name] is not None: value = params[param_name]
query_params[param_name] = params[param_name] # Skip empty values for optional params (LLMs sometimes
# pass "" instead of omitting optional parameters).
if value is None or (value == '' and not param.get('required')):
continue
query_params[param_name] = value
final_url = f'{url.rstrip("/")}{route_path}' final_url = f'{url.rstrip("/")}{route_path}'
for key, value in path_params.items(): for key, value in path_params.items():
@@ -1273,7 +1278,13 @@ async def execute_tool_server(
try: try:
response_data = await response.json() response_data = await response.json()
except Exception: except Exception:
response_data = await response.text() content_type = response.headers.get('Content-Type', '').split(';')[0].strip()
if content_type.startswith('text/') or not content_type:
response_data = await response.text()
else:
raw = await response.read()
b64 = base64.b64encode(raw).decode()
response_data = f'data:{content_type};base64,{b64}'
response_headers = response.headers response_headers = response.headers
return (response_data, response_headers) return (response_data, response_headers)
@@ -1292,7 +1303,13 @@ async def execute_tool_server(
try: try:
response_data = await response.json() response_data = await response.json()
except Exception: except Exception:
response_data = await response.text() content_type = response.headers.get('Content-Type', '').split(';')[0].strip()
if content_type.startswith('text/') or not content_type:
response_data = await response.text()
else:
raw = await response.read()
b64 = base64.b64encode(raw).decode()
response_data = f'data:{content_type};base64,{b64}'
response_headers = response.headers response_headers = response.headers
return (response_data, response_headers) return (response_data, response_headers)