From efe5416f8373455217572f50464860fdd9da63f3 Mon Sep 17 00:00:00 2001 From: Classic298 <27028174+Classic298@users.noreply.github.com> Date: Thu, 12 Feb 2026 01:29:10 +0100 Subject: [PATCH] fix: reduce TTFT by caching model lookups in chat completion (#20886) fix: reduce TTFT by caching model lookups in chat completion Skip expensive get_all_models() calls when models are already cached in app.state. This significantly reduces Time To First Token (TTFT) for chat completions and embeddings requests. Previously, every request called get_all_models() which fetches model lists from all configured backends. Now we check the cache first and only call get_all_models() on cache miss. Affected endpoints: - openai: generate_chat_completion, embeddings - ollama: embed, embeddings Fixes #20069 Co-authored-by: Michael <42099345+mickeytheseal@users.noreply.github.com> --- backend/open_webui/routers/ollama.py | 18 ++++++++++++------ backend/open_webui/routers/openai.py | 14 +++++++++++--- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/backend/open_webui/routers/ollama.py b/backend/open_webui/routers/ollama.py index b3f0e7ec9..d0b86a499 100644 --- a/backend/open_webui/routers/ollama.py +++ b/backend/open_webui/routers/ollama.py @@ -1027,14 +1027,17 @@ async def embed( log.info(f"generate_ollama_batch_embeddings {form_data}") if url_idx is None: - await get_all_models(request, user=user) - models = request.app.state.OLLAMA_MODELS - model = form_data.model if ":" not in model: model = f"{model}:latest" + # Check if model is already in app state cache to avoid expensive get_all_models() call + models = request.app.state.OLLAMA_MODELS + if not models or model not in models: + await get_all_models(request, user=user) + models = request.app.state.OLLAMA_MODELS + if model in models: url_idx = random.choice(models[model]["urls"]) else: @@ -1112,14 +1115,17 @@ async def embeddings( log.info(f"generate_ollama_embeddings {form_data}") if url_idx is None: - await get_all_models(request, user=user) - models = request.app.state.OLLAMA_MODELS - model = form_data.model if ":" not in model: model = f"{model}:latest" + # Check if model is already in app state cache to avoid expensive get_all_models() call + models = request.app.state.OLLAMA_MODELS + if not models or model not in models: + await get_all_models(request, user=user) + models = request.app.state.OLLAMA_MODELS + if model in models: url_idx = random.choice(models[model]["urls"]) else: diff --git a/backend/open_webui/routers/openai.py b/backend/open_webui/routers/openai.py index d5f3aa88b..5e0a74d61 100644 --- a/backend/open_webui/routers/openai.py +++ b/backend/open_webui/routers/openai.py @@ -991,8 +991,13 @@ async def generate_chat_completion( detail="Model not found", ) - await get_all_models(request, user=user) - model = request.app.state.OPENAI_MODELS.get(model_id) + # Check if model is already in app state cache to avoid expensive get_all_models() call + # This significantly reduces TTFT when models are already cached + model = request.app.state.OPENAI_MODELS.get(model_id) if request.app.state.OPENAI_MODELS else None + if not model: + await get_all_models(request, user=user) + model = request.app.state.OPENAI_MODELS.get(model_id) + if model: idx = model["urlIdx"] else: @@ -1148,9 +1153,12 @@ async def embeddings(request: Request, form_data: dict, user): # Prepare payload/body body = json.dumps(form_data) # Find correct backend url/key based on model - await get_all_models(request, user=user) model_id = form_data.get("model") + # Check if model is already in app state cache to avoid expensive get_all_models() call models = request.app.state.OPENAI_MODELS + if not models or model_id not in models: + await get_all_models(request, user=user) + models = request.app.state.OPENAI_MODELS if model_id in models: idx = models[model_id]["urlIdx"]