diff --git a/backend/open_webui/config.py b/backend/open_webui/config.py index 08a30815b..8294e5166 100644 --- a/backend/open_webui/config.py +++ b/backend/open_webui/config.py @@ -3320,6 +3320,12 @@ FIRECRAWL_API_BASE_URL = PersistentConfig( os.environ.get("FIRECRAWL_API_BASE_URL", "https://api.firecrawl.dev"), ) +FIRECRAWL_TIMEOUT = PersistentConfig( + "FIRECRAWL_TIMEOUT", + "rag.web.loader.firecrawl_timeout", + os.environ.get("FIRECRAWL_TIMEOUT", ""), +) + EXTERNAL_WEB_SEARCH_URL = PersistentConfig( "EXTERNAL_WEB_SEARCH_URL", "rag.web.search.external_web_search_url", diff --git a/backend/open_webui/main.py b/backend/open_webui/main.py index 68c90c7ee..952ff7f2e 100644 --- a/backend/open_webui/main.py +++ b/backend/open_webui/main.py @@ -208,6 +208,7 @@ from open_webui.config import ( PLAYWRIGHT_TIMEOUT, FIRECRAWL_API_BASE_URL, FIRECRAWL_API_KEY, + FIRECRAWL_TIMEOUT, WEB_LOADER_ENGINE, WEB_LOADER_CONCURRENT_REQUESTS, WEB_LOADER_TIMEOUT, @@ -987,6 +988,7 @@ app.state.config.PLAYWRIGHT_WS_URL = PLAYWRIGHT_WS_URL app.state.config.PLAYWRIGHT_TIMEOUT = PLAYWRIGHT_TIMEOUT app.state.config.FIRECRAWL_API_BASE_URL = FIRECRAWL_API_BASE_URL app.state.config.FIRECRAWL_API_KEY = FIRECRAWL_API_KEY +app.state.config.FIRECRAWL_TIMEOUT = FIRECRAWL_TIMEOUT app.state.config.TAVILY_EXTRACT_DEPTH = TAVILY_EXTRACT_DEPTH app.state.EMBEDDING_FUNCTION = None diff --git a/backend/open_webui/retrieval/web/utils.py b/backend/open_webui/retrieval/web/utils.py index eb01976e9..6c1ea4b1b 100644 --- a/backend/open_webui/retrieval/web/utils.py +++ b/backend/open_webui/retrieval/web/utils.py @@ -36,6 +36,7 @@ from open_webui.config import ( WEB_LOADER_TIMEOUT, FIRECRAWL_API_BASE_URL, FIRECRAWL_API_KEY, + FIRECRAWL_TIMEOUT, TAVILY_API_KEY, TAVILY_EXTRACT_DEPTH, EXTERNAL_WEB_LOADER_URL, @@ -189,6 +190,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): continue_on_failure: bool = True, api_key: Optional[str] = None, api_url: Optional[str] = None, + timeout: Optional[int] = None, mode: Literal["crawl", "scrape", "map"] = "scrape", proxy: Optional[Dict[str, str]] = None, params: Optional[Dict] = None, @@ -231,6 +233,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): self.continue_on_failure = continue_on_failure self.api_key = api_key self.api_url = api_url + self.timeout = timeout self.mode = mode self.params = params or {} @@ -253,7 +256,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): ignore_invalid_urls=True, remove_base64_images=True, max_age=300000, # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values - wait_timeout=len(self.web_paths) * 3, + wait_timeout=self.timeout if self.timeout else len(self.web_paths) * 3, **self.params, ) @@ -294,7 +297,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin): ignore_invalid_urls=True, remove_base64_images=True, max_age=300000, # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values - wait_timeout=len(self.web_paths) * 3, + wait_timeout=self.timeout if self.timeout else len(self.web_paths) * 3, **self.params, ) @@ -697,6 +700,11 @@ def get_web_loader( WebLoaderClass = SafeFireCrawlLoader web_loader_args["api_key"] = FIRECRAWL_API_KEY.value web_loader_args["api_url"] = FIRECRAWL_API_BASE_URL.value + if FIRECRAWL_TIMEOUT.value: + try: + web_loader_args["timeout"] = int(FIRECRAWL_TIMEOUT.value) + except ValueError: + pass if WEB_LOADER_ENGINE.value == "tavily": WebLoaderClass = SafeTavilyLoader diff --git a/backend/open_webui/routers/retrieval.py b/backend/open_webui/routers/retrieval.py index f35c6b43b..b93ea9ad8 100644 --- a/backend/open_webui/routers/retrieval.py +++ b/backend/open_webui/routers/retrieval.py @@ -564,6 +564,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)): "PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT, "FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY, "FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL, + "FIRECRAWL_TIMEOUT": request.app.state.config.FIRECRAWL_TIMEOUT, "TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH, "EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL, "EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY, @@ -624,6 +625,7 @@ class WebConfig(BaseModel): PLAYWRIGHT_TIMEOUT: Optional[int] = None FIRECRAWL_API_KEY: Optional[str] = None FIRECRAWL_API_BASE_URL: Optional[str] = None + FIRECRAWL_TIMEOUT: Optional[str] = None TAVILY_EXTRACT_DEPTH: Optional[str] = None EXTERNAL_WEB_SEARCH_URL: Optional[str] = None EXTERNAL_WEB_SEARCH_API_KEY: Optional[str] = None @@ -1123,6 +1125,7 @@ async def update_rag_config( request.app.state.config.FIRECRAWL_API_BASE_URL = ( form_data.web.FIRECRAWL_API_BASE_URL ) + request.app.state.config.FIRECRAWL_TIMEOUT = form_data.web.FIRECRAWL_TIMEOUT request.app.state.config.EXTERNAL_WEB_SEARCH_URL = ( form_data.web.EXTERNAL_WEB_SEARCH_URL ) @@ -1259,6 +1262,7 @@ async def update_rag_config( "PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT, "FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY, "FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL, + "FIRECRAWL_TIMEOUT": request.app.state.config.FIRECRAWL_TIMEOUT, "TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH, "EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL, "EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY, diff --git a/src/lib/components/admin/Settings/WebSearch.svelte b/src/lib/components/admin/Settings/WebSearch.svelte index 6c9035f3c..dc679bbe8 100644 --- a/src/lib/components/admin/Settings/WebSearch.svelte +++ b/src/lib/components/admin/Settings/WebSearch.svelte @@ -628,6 +628,24 @@ bind:value={webConfig.FIRECRAWL_API_KEY} /> + +
+
+ {$i18n.t('Firecrawl Timeout (s)')} +
+ +
+
+ +
+
+
{:else if webConfig.WEB_SEARCH_ENGINE === 'external'}