Emil_Shanaty/open-webui-ai4me
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #12822 from tth37/feat_external_search_loader

Browse Source 
feat: Support for Self-Hosted/External Web Search/Loader Engines
This commit is contained in:
Tim Jaeryang Baek
2025-04-18 23:51:27 -07:00
committed by GitHub
parent 1c18282d19 dcb3f18e1e
commit 87844a8042
7 changed files with 237 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/routers/retrieval.py
+33
View File
@@ -61,6 +61,7 @@ from open_webui.retrieval.web.bing import search_bing
from open_webui.retrieval.web.exa import search_exa
from open_webui.retrieval.web.perplexity import search_perplexity
from open_webui.retrieval.web.sougou import search_sougou
from open_webui.retrieval.web.external import search_external
from open_webui.retrieval.utils import (
get_embedding_function,
@@ -418,6 +419,10 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
"FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY,
"FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL,
"TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH,
"EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
"EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
"EXTERNAL_WEB_LOADER_URL": request.app.state.config.EXTERNAL_WEB_LOADER_URL,
"EXTERNAL_WEB_LOADER_API_KEY": request.app.state.config.EXTERNAL_WEB_LOADER_API_KEY,
"YOUTUBE_LOADER_LANGUAGE": request.app.state.config.YOUTUBE_LOADER_LANGUAGE,
"YOUTUBE_LOADER_PROXY_URL": request.app.state.config.YOUTUBE_LOADER_PROXY_URL,
"YOUTUBE_LOADER_TRANSLATION": request.app.state.YOUTUBE_LOADER_TRANSLATION,
@@ -463,6 +468,10 @@ class WebConfig(BaseModel):
FIRECRAWL_API_KEY: Optional[str] = None
FIRECRAWL_API_BASE_URL: Optional[str] = None
TAVILY_EXTRACT_DEPTH: Optional[str] = None
EXTERNAL_WEB_SEARCH_URL: Optional[str] = None
EXTERNAL_WEB_SEARCH_API_KEY: Optional[str] = None
EXTERNAL_WEB_LOADER_URL: Optional[str] = None
EXTERNAL_WEB_LOADER_API_KEY: Optional[str] = None
YOUTUBE_LOADER_LANGUAGE: Optional[List[str]] = None
YOUTUBE_LOADER_PROXY_URL: Optional[str] = None
YOUTUBE_LOADER_TRANSLATION: Optional[str] = None
@@ -697,6 +706,18 @@ async def update_rag_config(
request.app.state.config.FIRECRAWL_API_BASE_URL = (
form_data.web.FIRECRAWL_API_BASE_URL
)
request.app.state.config.EXTERNAL_WEB_SEARCH_URL = (
form_data.web.EXTERNAL_WEB_SEARCH_URL
)
request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY = (
form_data.web.EXTERNAL_WEB_SEARCH_API_KEY
)
request.app.state.config.EXTERNAL_WEB_LOADER_URL = (
form_data.web.EXTERNAL_WEB_LOADER_URL
)
request.app.state.config.EXTERNAL_WEB_LOADER_API_KEY = (
form_data.web.EXTERNAL_WEB_LOADER_API_KEY
)
request.app.state.config.TAVILY_EXTRACT_DEPTH = (
form_data.web.TAVILY_EXTRACT_DEPTH
)
@@ -778,6 +799,10 @@ async def update_rag_config(
"FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY,
"FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL,
"TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH,
"EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
"EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
"EXTERNAL_WEB_LOADER_URL": request.app.state.config.EXTERNAL_WEB_LOADER_URL,
"EXTERNAL_WEB_LOADER_API_KEY": request.app.state.config.EXTERNAL_WEB_LOADER_API_KEY,
"YOUTUBE_LOADER_LANGUAGE": request.app.state.config.YOUTUBE_LOADER_LANGUAGE,
"YOUTUBE_LOADER_PROXY_URL": request.app.state.config.YOUTUBE_LOADER_PROXY_URL,
"YOUTUBE_LOADER_TRANSLATION": request.app.state.YOUTUBE_LOADER_TRANSLATION,
@@ -1465,6 +1490,14 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
raise Exception(
"No SOUGOU_API_SID or SOUGOU_API_SK found in environment variables"
)
elif engine == "external":
return search_external(
request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
query,
request.app.state.config.WEB_SEARCH_RESULT_COUNT,
request.app.state.config.WEB_SEARCH_DOMAIN_FILTER_LIST,
)
else:
raise Exception("No search engine API key found in environment variables")