refactor(firecrawl): use v2 API directly (#23934)

Co-authored-by: Tim Baek <tim@openwebui.com>

backend/open_webui/retrieval/web/utils.py

@@ -30,6 +30,7 @@ from langchain_core.documents import Document
 
 from open_webui.retrieval.loaders.tavily import TavilyLoader
 from open_webui.retrieval.loaders.external_web import ExternalWebLoader
+from open_webui.retrieval.web.firecrawl import scrape_firecrawl_url
 from open_webui.constants import ERROR_MESSAGES
 from open_webui.config import (
     ENABLE_RAG_LOCAL_WEB_FETCH,
@@ -218,39 +219,20 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
 
     def lazy_load(self) -> Iterator[Document]:
         try:
-            headers = {
-                'Content-Type': 'application/json',
-                'Authorization': f'Bearer {self.api_key}',
-            }
-
             for url in self.web_paths:
-                payload = {
-                    'url': url,
-                    'formats': ['markdown'],
-                    **self.params,
-                }
-                if self.timeout:
-                    payload['timeout'] = self.timeout * 1000
-
-                response = requests.post(
-                    f'{self.api_url}/v1/scrape',
-                    headers=headers,
-                    json=payload,
-                    timeout=self.timeout or 60,
-                    verify=self.verify_ssl,
-                )
-                response.raise_for_status()
-                data = response.json().get('data', {})
-                metadata = data.get('metadata', {})
-                source = metadata.get('url') or metadata.get('sourceURL') or url
-
-                yield Document(
-                    page_content=data.get('markdown', ''),
-                    metadata={'source': source},
+                doc = scrape_firecrawl_url(
+                    self.api_url,
+                    self.api_key,
+                    url,
+                    verify_ssl=self.verify_ssl,
+                    timeout=self.timeout,
+                    params=self.params,
                 )
+                if doc is not None:
+                    yield doc
         except Exception as e:
             if self.continue_on_failure:
-                log.exception(f'Error extracting content from URLs: {e}')
+                log.warning(f'Error extracting content from URLs with Firecrawl: {e}')
             else:
                 raise e
 
@@ -261,7 +243,7 @@ class SafeFireCrawlLoader(BaseLoader, RateLimitMixin, URLProcessingMixin):
                 yield doc
         except Exception as e:
             if self.continue_on_failure:
-                log.exception(f'Error extracting content from URLs: {e}')
+                log.warning(f'Error extracting content from URLs with Firecrawl: {e}')
             else:
                 raise e