Emil_Shanaty/open-webui-ai4me
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

enh: FIRECRAWL_TIMEOUT

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2026-01-01 02:07:22 +04:00
parent b619a157bc
commit 89ad1c68d1
5 changed files with 40 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/config.py
+6
View File
@@ -3320,6 +3320,12 @@ FIRECRAWL_API_BASE_URL = PersistentConfig(
os.environ.get("FIRECRAWL_API_BASE_URL", "https://api.firecrawl.dev"),
)
FIRECRAWL_TIMEOUT = PersistentConfig(
"FIRECRAWL_TIMEOUT",
"rag.web.loader.firecrawl_timeout",
os.environ.get("FIRECRAWL_TIMEOUT", ""),
)
EXTERNAL_WEB_SEARCH_URL = PersistentConfig(
"EXTERNAL_WEB_SEARCH_URL",
"rag.web.search.external_web_search_url",
backend/open_webui/main.py
+2
View File
@@ -208,6 +208,7 @@ from open_webui.config import (
PLAYWRIGHT_TIMEOUT,
FIRECRAWL_API_BASE_URL,
FIRECRAWL_API_KEY,
FIRECRAWL_TIMEOUT,
WEB_LOADER_ENGINE,
WEB_LOADER_CONCURRENT_REQUESTS,
WEB_LOADER_TIMEOUT,
@@ -987,6 +988,7 @@ app.state.config.PLAYWRIGHT_WS_URL = PLAYWRIGHT_WS_URL
app.state.config.PLAYWRIGHT_TIMEOUT = PLAYWRIGHT_TIMEOUT
app.state.config.FIRECRAWL_API_BASE_URL = FIRECRAWL_API_BASE_URL
app.state.config.FIRECRAWL_API_KEY = FIRECRAWL_API_KEY
app.state.config.FIRECRAWL_TIMEOUT = FIRECRAWL_TIMEOUT
app.state.config.TAVILY_EXTRACT_DEPTH = TAVILY_EXTRACT_DEPTH
app.state.EMBEDDING_FUNCTION = None
backend/open_webui/retrieval/web/utils.py
+10 -2
View File
@@ -36,6 +36,7 @@ from open_webui.config import (
WEB_LOADER_TIMEOUT,
FIRECRAWL_API_BASE_URL,
FIRECRAWL_API_KEY,
FIRECRAWL_TIMEOUT,
TAVILY_API_KEY,
TAVILY_EXTRACT_DEPTH,
EXTERNAL_WEB_LOADER_URL,
@@ -189,6 +190,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
continue_on_failure: bool = True,
api_key: Optional[str] = None,
api_url: Optional[str] = None,
timeout: Optional[int] = None,
mode: Literal["crawl", "scrape", "map"] = "scrape",
proxy: Optional[Dict[str, str]] = None,
params: Optional[Dict] = None,
@@ -231,6 +233,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
self.continue_on_failure = continue_on_failure
self.api_key = api_key
self.api_url = api_url
self.timeout = timeout
self.mode = mode
self.params = params or {}
@@ -253,7 +256,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
ignore_invalid_urls=True,
remove_base64_images=True,
max_age=300000, # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
wait_timeout=len(self.web_paths) * 3,
wait_timeout=self.timeout if self.timeout else len(self.web_paths) * 3,
**self.params,
)
@@ -294,7 +297,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
ignore_invalid_urls=True,
remove_base64_images=True,
max_age=300000, # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
wait_timeout=len(self.web_paths) * 3,
wait_timeout=self.timeout if self.timeout else len(self.web_paths) * 3,
**self.params,
)
@@ -697,6 +700,11 @@ def get_web_loader(
WebLoaderClass = SafeFireCrawlLoader
web_loader_args["api_key"] = FIRECRAWL_API_KEY.value
web_loader_args["api_url"] = FIRECRAWL_API_BASE_URL.value
if FIRECRAWL_TIMEOUT.value:
try:
web_loader_args["timeout"] = int(FIRECRAWL_TIMEOUT.value)
except ValueError:
pass
if WEB_LOADER_ENGINE.value == "tavily":
WebLoaderClass = SafeTavilyLoader
backend/open_webui/routers/retrieval.py
+4
View File
@@ -564,6 +564,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
"PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT,
"FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY,
"FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL,
"FIRECRAWL_TIMEOUT": request.app.state.config.FIRECRAWL_TIMEOUT,
"TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH,
"EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
"EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
@@ -624,6 +625,7 @@ class WebConfig(BaseModel):
PLAYWRIGHT_TIMEOUT: Optional[int] = None
FIRECRAWL_API_KEY: Optional[str] = None
FIRECRAWL_API_BASE_URL: Optional[str] = None
FIRECRAWL_TIMEOUT: Optional[str] = None
TAVILY_EXTRACT_DEPTH: Optional[str] = None
EXTERNAL_WEB_SEARCH_URL: Optional[str] = None
EXTERNAL_WEB_SEARCH_API_KEY: Optional[str] = None
@@ -1123,6 +1125,7 @@ async def update_rag_config(
request.app.state.config.FIRECRAWL_API_BASE_URL = (
form_data.web.FIRECRAWL_API_BASE_URL
)
request.app.state.config.FIRECRAWL_TIMEOUT = form_data.web.FIRECRAWL_TIMEOUT
request.app.state.config.EXTERNAL_WEB_SEARCH_URL = (
form_data.web.EXTERNAL_WEB_SEARCH_URL
)
@@ -1259,6 +1262,7 @@ async def update_rag_config(
"PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT,
"FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY,
"FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL,
"FIRECRAWL_TIMEOUT": request.app.state.config.FIRECRAWL_TIMEOUT,
"TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH,
"EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
"EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
src/lib/components/admin/Settings/WebSearch.svelte
+18
View File
@@ -628,6 +628,24 @@
bind:value={webConfig.FIRECRAWL_API_KEY}
/>
</div>
<div class="mt-2">
<div class=" self-center text-xs font-medium mb-1">
{$i18n.t('Firecrawl Timeout (s)')}
</div>
<div class="flex w-full">
<div class="flex-1">
<input
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
type="number"
placeholder={$i18n.t('Enter Firecrawl Timeout')}
bind:value={webConfig.FIRECRAWL_TIMEOUT}
autocomplete="off"
/>
</div>
</div>
</div>
</div>
{:else if webConfig.WEB_SEARCH_ENGINE === 'external'}
<div class="mb-2.5 flex w-full flex-col">