Implement domain whitelisting for web search results

This commit is contained in:
Que Nguyen
2024-06-13 07:14:48 +07:00
committed by GitHub
parent a382e82dec
commit 7b5f434a07
10 changed files with 63 additions and 29 deletions
+5 -4
View File
@@ -1,15 +1,15 @@
import logging
from typing import List
import requests
from apps.rag.search.main import SearchResult
from apps.rag.search.main import SearchResult, filter_by_whitelist
from config import SRC_LOG_LEVELS
log = logging.getLogger(__name__)
log.setLevel(SRC_LOG_LEVELS["RAG"])
def search_brave(api_key: str, query: str, count: int) -> list[SearchResult]:
def search_brave(api_key: str, query: str, whitelist:List[str], count: int) -> list[SearchResult]:
"""Search using Brave's Search API and return the results as a list of SearchResult objects.
Args:
@@ -29,9 +29,10 @@ def search_brave(api_key: str, query: str, count: int) -> list[SearchResult]:
json_response = response.json()
results = json_response.get("web", {}).get("results", [])
filtered_results = filter_by_whitelist(results, whitelist)
return [
SearchResult(
link=result["url"], title=result.get("title"), snippet=result.get("snippet")
)
for result in results[:count]
for result in filtered_results[:count]
]