enh: FIRECRAWL_TIMEOUT

backend/open_webui/retrieval/web/utils.py

@@ -36,6 +36,7 @@ from open_webui.config import (
     WEB_LOADER_TIMEOUT,
     FIRECRAWL_API_BASE_URL,
     FIRECRAWL_API_KEY,
+    FIRECRAWL_TIMEOUT,
     TAVILY_API_KEY,
     TAVILY_EXTRACT_DEPTH,
     EXTERNAL_WEB_LOADER_URL,
@@ -189,6 +190,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
         continue_on_failure: bool = True,
         api_key: Optional[str] = None,
         api_url: Optional[str] = None,
+        timeout: Optional[int] = None,
         mode: Literal["crawl", "scrape", "map"] = "scrape",
         proxy: Optional[Dict[str, str]] = None,
         params: Optional[Dict] = None,
@@ -231,6 +233,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
         self.continue_on_failure = continue_on_failure
         self.api_key = api_key
         self.api_url = api_url
+        self.timeout = timeout
         self.mode = mode
         self.params = params or {}
 
@@ -253,7 +256,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
                 ignore_invalid_urls=True,
                 remove_base64_images=True,
                 max_age=300000,  # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
-                wait_timeout=len(self.web_paths) * 3,
+                wait_timeout=self.timeout if self.timeout else len(self.web_paths) * 3,
                 **self.params,
             )
 
@@ -294,7 +297,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
                 ignore_invalid_urls=True,
                 remove_base64_images=True,
                 max_age=300000,  # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
-                wait_timeout=len(self.web_paths) * 3,
+                wait_timeout=self.timeout if self.timeout else len(self.web_paths) * 3,
                 **self.params,
             )
 
@@ -697,6 +700,11 @@ def get_web_loader(
         WebLoaderClass = SafeFireCrawlLoader
         web_loader_args["api_key"] = FIRECRAWL_API_KEY.value
         web_loader_args["api_url"] = FIRECRAWL_API_BASE_URL.value
+        if FIRECRAWL_TIMEOUT.value:
+            try:
+                web_loader_args["timeout"] = int(FIRECRAWL_TIMEOUT.value)
+            except ValueError:
+                pass
 
     if WEB_LOADER_ENGINE.value == "tavily":
         WebLoaderClass = SafeTavilyLoader