Emil_Shanaty/open-webui-ai4me
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add mineru as document parser support with both local and managed api

Browse Source
This commit is contained in:
palazski
2025-10-13 21:09:52 +03:00
parent 46ae3f4f5d
commit 40e9d9c330
6 changed files with 866 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/routers/retrieval.py
+87
View File
@@ -466,6 +466,16 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
"DOCUMENT_INTELLIGENCE_ENDPOINT": request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT,
"DOCUMENT_INTELLIGENCE_KEY": request.app.state.config.DOCUMENT_INTELLIGENCE_KEY,
"MISTRAL_OCR_API_KEY": request.app.state.config.MISTRAL_OCR_API_KEY,
# MinerU settings
"MINERU_API_MODE": request.app.state.config.MINERU_API_MODE,
"MINERU_API_URL": request.app.state.config.MINERU_API_URL,
"MINERU_API_KEY": request.app.state.config.MINERU_API_KEY,
"MINERU_ENABLE_OCR": request.app.state.config.MINERU_ENABLE_OCR,
"MINERU_ENABLE_FORMULA": request.app.state.config.MINERU_ENABLE_FORMULA,
"MINERU_ENABLE_TABLE": request.app.state.config.MINERU_ENABLE_TABLE,
"MINERU_LANGUAGE": request.app.state.config.MINERU_LANGUAGE,
"MINERU_MODEL_VERSION": request.app.state.config.MINERU_MODEL_VERSION,
"MINERU_PAGE_RANGES": request.app.state.config.MINERU_PAGE_RANGES,
# Reranking settings
"RAG_RERANKING_MODEL": request.app.state.config.RAG_RERANKING_MODEL,
"RAG_RERANKING_ENGINE": request.app.state.config.RAG_RERANKING_ENGINE,
@@ -647,6 +657,17 @@ class ConfigForm(BaseModel):
DOCUMENT_INTELLIGENCE_KEY: Optional[str] = None
MISTRAL_OCR_API_KEY: Optional[str] = None
# MinerU settings
MINERU_API_MODE: Optional[str] = None
MINERU_API_URL: Optional[str] = None
MINERU_API_KEY: Optional[str] = None
MINERU_ENABLE_OCR: Optional[bool] = None
MINERU_ENABLE_FORMULA: Optional[bool] = None
MINERU_ENABLE_TABLE: Optional[bool] = None
MINERU_LANGUAGE: Optional[str] = None
MINERU_MODEL_VERSION: Optional[str] = None
MINERU_PAGE_RANGES: Optional[str] = None
# Reranking settings
RAG_RERANKING_MODEL: Optional[str] = None
RAG_RERANKING_ENGINE: Optional[str] = None
@@ -886,6 +907,53 @@ async def update_rag_config(
else request.app.state.config.MISTRAL_OCR_API_KEY
)
# MinerU settings
request.app.state.config.MINERU_API_MODE = (
form_data.MINERU_API_MODE
if form_data.MINERU_API_MODE is not None
else request.app.state.config.MINERU_API_MODE
)
request.app.state.config.MINERU_API_URL = (
form_data.MINERU_API_URL
if form_data.MINERU_API_URL is not None
else request.app.state.config.MINERU_API_URL
)
request.app.state.config.MINERU_API_KEY = (
form_data.MINERU_API_KEY
if form_data.MINERU_API_KEY is not None
else request.app.state.config.MINERU_API_KEY
)
request.app.state.config.MINERU_ENABLE_OCR = (
form_data.MINERU_ENABLE_OCR
if form_data.MINERU_ENABLE_OCR is not None
else request.app.state.config.MINERU_ENABLE_OCR
)
request.app.state.config.MINERU_ENABLE_FORMULA = (
form_data.MINERU_ENABLE_FORMULA
if form_data.MINERU_ENABLE_FORMULA is not None
else request.app.state.config.MINERU_ENABLE_FORMULA
)
request.app.state.config.MINERU_ENABLE_TABLE = (
form_data.MINERU_ENABLE_TABLE
if form_data.MINERU_ENABLE_TABLE is not None
else request.app.state.config.MINERU_ENABLE_TABLE
)
request.app.state.config.MINERU_LANGUAGE = (
form_data.MINERU_LANGUAGE
if form_data.MINERU_LANGUAGE is not None
else request.app.state.config.MINERU_LANGUAGE
)
request.app.state.config.MINERU_MODEL_VERSION = (
form_data.MINERU_MODEL_VERSION
if form_data.MINERU_MODEL_VERSION is not None
else request.app.state.config.MINERU_MODEL_VERSION
)
request.app.state.config.MINERU_PAGE_RANGES = (
form_data.MINERU_PAGE_RANGES
if form_data.MINERU_PAGE_RANGES is not None
else request.app.state.config.MINERU_PAGE_RANGES
)
# Reranking settings
if request.app.state.config.RAG_RERANKING_ENGINE == "":
# Unloading the internal reranker and clear VRAM memory
@@ -1150,6 +1218,16 @@ async def update_rag_config(
"DOCUMENT_INTELLIGENCE_ENDPOINT": request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT,
"DOCUMENT_INTELLIGENCE_KEY": request.app.state.config.DOCUMENT_INTELLIGENCE_KEY,
"MISTRAL_OCR_API_KEY": request.app.state.config.MISTRAL_OCR_API_KEY,
# MinerU settings
"MINERU_API_MODE": request.app.state.config.MINERU_API_MODE,
"MINERU_API_URL": request.app.state.config.MINERU_API_URL,
"MINERU_API_KEY": request.app.state.config.MINERU_API_KEY,
"MINERU_ENABLE_OCR": request.app.state.config.MINERU_ENABLE_OCR,
"MINERU_ENABLE_FORMULA": request.app.state.config.MINERU_ENABLE_FORMULA,
"MINERU_ENABLE_TABLE": request.app.state.config.MINERU_ENABLE_TABLE,
"MINERU_LANGUAGE": request.app.state.config.MINERU_LANGUAGE,
"MINERU_MODEL_VERSION": request.app.state.config.MINERU_MODEL_VERSION,
"MINERU_PAGE_RANGES": request.app.state.config.MINERU_PAGE_RANGES,
# Reranking settings
"RAG_RERANKING_MODEL": request.app.state.config.RAG_RERANKING_MODEL,
"RAG_RERANKING_ENGINE": request.app.state.config.RAG_RERANKING_ENGINE,
@@ -1560,6 +1638,15 @@ def process_file(
DOCUMENT_INTELLIGENCE_ENDPOINT=request.app.state.config.DOCUMENT_INTELLIGENCE_ENDPOINT,
DOCUMENT_INTELLIGENCE_KEY=request.app.state.config.DOCUMENT_INTELLIGENCE_KEY,
MISTRAL_OCR_API_KEY=request.app.state.config.MISTRAL_OCR_API_KEY,
MINERU_API_MODE=request.app.state.config.MINERU_API_MODE,
MINERU_API_URL=request.app.state.config.MINERU_API_URL,
MINERU_API_KEY=request.app.state.config.MINERU_API_KEY,
MINERU_ENABLE_OCR=request.app.state.config.MINERU_ENABLE_OCR,
MINERU_ENABLE_FORMULA=request.app.state.config.MINERU_ENABLE_FORMULA,
MINERU_ENABLE_TABLE=request.app.state.config.MINERU_ENABLE_TABLE,
MINERU_LANGUAGE=request.app.state.config.MINERU_LANGUAGE,
MINERU_MODEL_VERSION=request.app.state.config.MINERU_MODEL_VERSION,
MINERU_PAGE_RANGES=request.app.state.config.MINERU_PAGE_RANGES,
)
docs = loader.load(
file.filename, file.meta.get("content_type"), file_path