enh: FIRECRAWL_TIMEOUT
This commit is contained in:
@@ -3320,6 +3320,12 @@ FIRECRAWL_API_BASE_URL = PersistentConfig(
|
|||||||
os.environ.get("FIRECRAWL_API_BASE_URL", "https://api.firecrawl.dev"),
|
os.environ.get("FIRECRAWL_API_BASE_URL", "https://api.firecrawl.dev"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
FIRECRAWL_TIMEOUT = PersistentConfig(
|
||||||
|
"FIRECRAWL_TIMEOUT",
|
||||||
|
"rag.web.loader.firecrawl_timeout",
|
||||||
|
os.environ.get("FIRECRAWL_TIMEOUT", ""),
|
||||||
|
)
|
||||||
|
|
||||||
EXTERNAL_WEB_SEARCH_URL = PersistentConfig(
|
EXTERNAL_WEB_SEARCH_URL = PersistentConfig(
|
||||||
"EXTERNAL_WEB_SEARCH_URL",
|
"EXTERNAL_WEB_SEARCH_URL",
|
||||||
"rag.web.search.external_web_search_url",
|
"rag.web.search.external_web_search_url",
|
||||||
|
|||||||
@@ -208,6 +208,7 @@ from open_webui.config import (
|
|||||||
PLAYWRIGHT_TIMEOUT,
|
PLAYWRIGHT_TIMEOUT,
|
||||||
FIRECRAWL_API_BASE_URL,
|
FIRECRAWL_API_BASE_URL,
|
||||||
FIRECRAWL_API_KEY,
|
FIRECRAWL_API_KEY,
|
||||||
|
FIRECRAWL_TIMEOUT,
|
||||||
WEB_LOADER_ENGINE,
|
WEB_LOADER_ENGINE,
|
||||||
WEB_LOADER_CONCURRENT_REQUESTS,
|
WEB_LOADER_CONCURRENT_REQUESTS,
|
||||||
WEB_LOADER_TIMEOUT,
|
WEB_LOADER_TIMEOUT,
|
||||||
@@ -987,6 +988,7 @@ app.state.config.PLAYWRIGHT_WS_URL = PLAYWRIGHT_WS_URL
|
|||||||
app.state.config.PLAYWRIGHT_TIMEOUT = PLAYWRIGHT_TIMEOUT
|
app.state.config.PLAYWRIGHT_TIMEOUT = PLAYWRIGHT_TIMEOUT
|
||||||
app.state.config.FIRECRAWL_API_BASE_URL = FIRECRAWL_API_BASE_URL
|
app.state.config.FIRECRAWL_API_BASE_URL = FIRECRAWL_API_BASE_URL
|
||||||
app.state.config.FIRECRAWL_API_KEY = FIRECRAWL_API_KEY
|
app.state.config.FIRECRAWL_API_KEY = FIRECRAWL_API_KEY
|
||||||
|
app.state.config.FIRECRAWL_TIMEOUT = FIRECRAWL_TIMEOUT
|
||||||
app.state.config.TAVILY_EXTRACT_DEPTH = TAVILY_EXTRACT_DEPTH
|
app.state.config.TAVILY_EXTRACT_DEPTH = TAVILY_EXTRACT_DEPTH
|
||||||
|
|
||||||
app.state.EMBEDDING_FUNCTION = None
|
app.state.EMBEDDING_FUNCTION = None
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ from open_webui.config import (
|
|||||||
WEB_LOADER_TIMEOUT,
|
WEB_LOADER_TIMEOUT,
|
||||||
FIRECRAWL_API_BASE_URL,
|
FIRECRAWL_API_BASE_URL,
|
||||||
FIRECRAWL_API_KEY,
|
FIRECRAWL_API_KEY,
|
||||||
|
FIRECRAWL_TIMEOUT,
|
||||||
TAVILY_API_KEY,
|
TAVILY_API_KEY,
|
||||||
TAVILY_EXTRACT_DEPTH,
|
TAVILY_EXTRACT_DEPTH,
|
||||||
EXTERNAL_WEB_LOADER_URL,
|
EXTERNAL_WEB_LOADER_URL,
|
||||||
@@ -189,6 +190,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
|||||||
continue_on_failure: bool = True,
|
continue_on_failure: bool = True,
|
||||||
api_key: Optional[str] = None,
|
api_key: Optional[str] = None,
|
||||||
api_url: Optional[str] = None,
|
api_url: Optional[str] = None,
|
||||||
|
timeout: Optional[int] = None,
|
||||||
mode: Literal["crawl", "scrape", "map"] = "scrape",
|
mode: Literal["crawl", "scrape", "map"] = "scrape",
|
||||||
proxy: Optional[Dict[str, str]] = None,
|
proxy: Optional[Dict[str, str]] = None,
|
||||||
params: Optional[Dict] = None,
|
params: Optional[Dict] = None,
|
||||||
@@ -231,6 +233,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
|||||||
self.continue_on_failure = continue_on_failure
|
self.continue_on_failure = continue_on_failure
|
||||||
self.api_key = api_key
|
self.api_key = api_key
|
||||||
self.api_url = api_url
|
self.api_url = api_url
|
||||||
|
self.timeout = timeout
|
||||||
self.mode = mode
|
self.mode = mode
|
||||||
self.params = params or {}
|
self.params = params or {}
|
||||||
|
|
||||||
@@ -253,7 +256,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
|||||||
ignore_invalid_urls=True,
|
ignore_invalid_urls=True,
|
||||||
remove_base64_images=True,
|
remove_base64_images=True,
|
||||||
max_age=300000, # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
|
max_age=300000, # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
|
||||||
wait_timeout=len(self.web_paths) * 3,
|
wait_timeout=self.timeout if self.timeout else len(self.web_paths) * 3,
|
||||||
**self.params,
|
**self.params,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -294,7 +297,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
|
|||||||
ignore_invalid_urls=True,
|
ignore_invalid_urls=True,
|
||||||
remove_base64_images=True,
|
remove_base64_images=True,
|
||||||
max_age=300000, # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
|
max_age=300000, # 5 minutes https://docs.firecrawl.dev/features/fast-scraping#common-maxage-values
|
||||||
wait_timeout=len(self.web_paths) * 3,
|
wait_timeout=self.timeout if self.timeout else len(self.web_paths) * 3,
|
||||||
**self.params,
|
**self.params,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -697,6 +700,11 @@ def get_web_loader(
|
|||||||
WebLoaderClass = SafeFireCrawlLoader
|
WebLoaderClass = SafeFireCrawlLoader
|
||||||
web_loader_args["api_key"] = FIRECRAWL_API_KEY.value
|
web_loader_args["api_key"] = FIRECRAWL_API_KEY.value
|
||||||
web_loader_args["api_url"] = FIRECRAWL_API_BASE_URL.value
|
web_loader_args["api_url"] = FIRECRAWL_API_BASE_URL.value
|
||||||
|
if FIRECRAWL_TIMEOUT.value:
|
||||||
|
try:
|
||||||
|
web_loader_args["timeout"] = int(FIRECRAWL_TIMEOUT.value)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
|
|
||||||
if WEB_LOADER_ENGINE.value == "tavily":
|
if WEB_LOADER_ENGINE.value == "tavily":
|
||||||
WebLoaderClass = SafeTavilyLoader
|
WebLoaderClass = SafeTavilyLoader
|
||||||
|
|||||||
@@ -564,6 +564,7 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
|
|||||||
"PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT,
|
"PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT,
|
||||||
"FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY,
|
"FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY,
|
||||||
"FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL,
|
"FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL,
|
||||||
|
"FIRECRAWL_TIMEOUT": request.app.state.config.FIRECRAWL_TIMEOUT,
|
||||||
"TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH,
|
"TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH,
|
||||||
"EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
|
"EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
|
||||||
"EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
|
"EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
|
||||||
@@ -624,6 +625,7 @@ class WebConfig(BaseModel):
|
|||||||
PLAYWRIGHT_TIMEOUT: Optional[int] = None
|
PLAYWRIGHT_TIMEOUT: Optional[int] = None
|
||||||
FIRECRAWL_API_KEY: Optional[str] = None
|
FIRECRAWL_API_KEY: Optional[str] = None
|
||||||
FIRECRAWL_API_BASE_URL: Optional[str] = None
|
FIRECRAWL_API_BASE_URL: Optional[str] = None
|
||||||
|
FIRECRAWL_TIMEOUT: Optional[str] = None
|
||||||
TAVILY_EXTRACT_DEPTH: Optional[str] = None
|
TAVILY_EXTRACT_DEPTH: Optional[str] = None
|
||||||
EXTERNAL_WEB_SEARCH_URL: Optional[str] = None
|
EXTERNAL_WEB_SEARCH_URL: Optional[str] = None
|
||||||
EXTERNAL_WEB_SEARCH_API_KEY: Optional[str] = None
|
EXTERNAL_WEB_SEARCH_API_KEY: Optional[str] = None
|
||||||
@@ -1123,6 +1125,7 @@ async def update_rag_config(
|
|||||||
request.app.state.config.FIRECRAWL_API_BASE_URL = (
|
request.app.state.config.FIRECRAWL_API_BASE_URL = (
|
||||||
form_data.web.FIRECRAWL_API_BASE_URL
|
form_data.web.FIRECRAWL_API_BASE_URL
|
||||||
)
|
)
|
||||||
|
request.app.state.config.FIRECRAWL_TIMEOUT = form_data.web.FIRECRAWL_TIMEOUT
|
||||||
request.app.state.config.EXTERNAL_WEB_SEARCH_URL = (
|
request.app.state.config.EXTERNAL_WEB_SEARCH_URL = (
|
||||||
form_data.web.EXTERNAL_WEB_SEARCH_URL
|
form_data.web.EXTERNAL_WEB_SEARCH_URL
|
||||||
)
|
)
|
||||||
@@ -1259,6 +1262,7 @@ async def update_rag_config(
|
|||||||
"PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT,
|
"PLAYWRIGHT_TIMEOUT": request.app.state.config.PLAYWRIGHT_TIMEOUT,
|
||||||
"FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY,
|
"FIRECRAWL_API_KEY": request.app.state.config.FIRECRAWL_API_KEY,
|
||||||
"FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL,
|
"FIRECRAWL_API_BASE_URL": request.app.state.config.FIRECRAWL_API_BASE_URL,
|
||||||
|
"FIRECRAWL_TIMEOUT": request.app.state.config.FIRECRAWL_TIMEOUT,
|
||||||
"TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH,
|
"TAVILY_EXTRACT_DEPTH": request.app.state.config.TAVILY_EXTRACT_DEPTH,
|
||||||
"EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
|
"EXTERNAL_WEB_SEARCH_URL": request.app.state.config.EXTERNAL_WEB_SEARCH_URL,
|
||||||
"EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
|
"EXTERNAL_WEB_SEARCH_API_KEY": request.app.state.config.EXTERNAL_WEB_SEARCH_API_KEY,
|
||||||
|
|||||||
@@ -628,6 +628,24 @@
|
|||||||
bind:value={webConfig.FIRECRAWL_API_KEY}
|
bind:value={webConfig.FIRECRAWL_API_KEY}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="mt-2">
|
||||||
|
<div class=" self-center text-xs font-medium mb-1">
|
||||||
|
{$i18n.t('Firecrawl Timeout (s)')}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex w-full">
|
||||||
|
<div class="flex-1">
|
||||||
|
<input
|
||||||
|
class="w-full rounded-lg py-2 px-4 text-sm bg-gray-50 dark:text-gray-300 dark:bg-gray-850 outline-hidden"
|
||||||
|
type="number"
|
||||||
|
placeholder={$i18n.t('Enter Firecrawl Timeout')}
|
||||||
|
bind:value={webConfig.FIRECRAWL_TIMEOUT}
|
||||||
|
autocomplete="off"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{:else if webConfig.WEB_SEARCH_ENGINE === 'external'}
|
{:else if webConfig.WEB_SEARCH_ENGINE === 'external'}
|
||||||
<div class="mb-2.5 flex w-full flex-col">
|
<div class="mb-2.5 flex w-full flex-col">
|
||||||
|
|||||||
Reference in New Issue
Block a user