Emil_Shanaty/open-webui-ai4me
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stream delta chunk

Browse Source 
Co-Authored-By: Jan Kessler <Ithanil@users.noreply.github.com>
This commit is contained in:
Timothy Jaeryang Baek
2025-08-09 23:43:27 +04:00
co-authored by Jan Kessler
parent e2b2e8b245
commit 1a93891d97
5 changed files with 115 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/main.py
+25 -10
View File
@@ -1401,6 +1401,16 @@ async def chat_completion(
request.state.direct = True
request.state.model = model
# Chat Params
stream_delta_chunk_size = form_data.get("params", {}).get(
"stream_delta_chunk_size"
)
# Model Params
if model_info and model_info.params:
stream_delta_chunk_size = model_info.params.model_dump().get(
"stream_delta_chunk_size"
)
metadata = {
"user_id": user.id,
"chat_id": form_data.pop("chat_id", None),
@@ -1414,16 +1424,21 @@ async def chat_completion(
"variables": form_data.get("variables", {}),
"model": model,
"direct": model_item.get("direct", False),
**(
{"function_calling": "native"}
if form_data.get("params", {}).get("function_calling") == "native"
or (
model_info
and model_info.params.model_dump().get("function_calling")
== "native"
)
else {}
),
"params": {
"stream_delta_chunk_size": stream_delta_chunk_size,
"function_calling": (
"native"
if (
form_data.get("params", {}).get("function_calling") == "native"
or (
model_info
and model_info.params.model_dump().get("function_calling")
== "native"
)
)
else "default"
),
},
}
if metadata.get("chat_id") and (user and user.role != "admin"):
backend/open_webui/utils/middleware.py
+28 -7
View File
@@ -684,6 +684,7 @@ def apply_params_to_form_data(form_data, model):
open_webui_params = {
"stream_response": bool,
"stream_delta_chunk_size": int,
"function_calling": str,
"system": str,
}
@@ -930,7 +931,7 @@ async def process_chat_payload(request, form_data, user, metadata, model):
}
if tools_dict:
if metadata.get("function_calling") == "native":
if metadata.get("params", {}).get("function_calling") == "native":
# If the function calling is native, then call the tools function calling handler
metadata["tools"] = tools_dict
form_data["tools"] = [
@@ -1816,6 +1817,15 @@ async def process_chat_response(
response_tool_calls = []
delta_count = 0
delta_chunk_size = max(
1,
int(
metadata.get("params", {}).get("stream_delta_chunk_size")
or 1
),
)
async for line in response.body_iterator:
line = line.decode("utf-8") if isinstance(line, bytes) else line
data = line
@@ -2063,12 +2073,23 @@ async def process_chat_response(
),
}
await event_emitter(
{
"type": "chat:completion",
"data": data,
}
)
if delta:
delta_count += 1
if delta_count >= delta_chunk_size:
await event_emitter(
{
"type": "chat:completion",
"data": data,
}
)
delta_count = 0
else:
await event_emitter(
{
"type": "chat:completion",
"data": data,
}
)
except Exception as e:
done = "data: [DONE]" in line
if done:
backend/open_webui/utils/payload.py
+1
View File
@@ -69,6 +69,7 @@ def remove_open_webui_params(params: dict) -> dict:
"""
open_webui_params = {
"stream_response": bool,
"stream_delta_chunk_size": int,
"function_calling": str,
"system": str,
}