Merge branch 'dev' into main

2025-04-11 22:14:41 +12:00
parent b4465987f9 5fea0a2221
commit a8bbaa61bd
96 changed files with 1848 additions and 905 deletions
@@ -50,6 +50,8 @@ router = APIRouter()
 # Constants
 MAX_FILE_SIZE_MB = 25
 MAX_FILE_SIZE = MAX_FILE_SIZE_MB * 1024 * 1024  # Convert MB to bytes
+AZURE_MAX_FILE_SIZE_MB = 200
+AZURE_MAX_FILE_SIZE = AZURE_MAX_FILE_SIZE_MB * 1024 * 1024  # Convert MB to bytes

 log = logging.getLogger(__name__)
 log.setLevel(SRC_LOG_LEVELS["AUDIO"])
@@ -68,8 +70,8 @@ from pydub import AudioSegment
 from pydub.utils import mediainfo


-def is_mp4_audio(file_path):
-    """Check if the given file is an MP4 audio file."""
+def get_audio_format(file_path):
+    """Check if the given file needs to be converted to a different format."""
    if not os.path.isfile(file_path):
        log.error(f"File not found: {file_path}")
        return False
@@ -80,13 +82,17 @@ def is_mp4_audio(file_path):
        and info.get("codec_type") == "audio"
        and info.get("codec_tag_string") == "mp4a"
    ):
-        return True
-    return False
+        return "mp4"
+    elif info.get("format_name") == "ogg":
+        return "ogg"
+    elif info.get("format_name") == "matroska,webm":
+        return "webm"
+    return None


-def convert_mp4_to_wav(file_path, output_path):
-    """Convert MP4 audio file to WAV format."""
-    audio = AudioSegment.from_file(file_path, format="mp4")
+def convert_audio_to_wav(file_path, output_path, conversion_type):
+    """Convert MP4/OGG audio file to WAV format."""
+    audio = AudioSegment.from_file(file_path, format=conversion_type)
    audio.export(output_path, format="wav")
    log.info(f"Converted {file_path} to {output_path}")

@@ -141,6 +147,9 @@ class STTConfigForm(BaseModel):
    MODEL: str
    WHISPER_MODEL: str
    DEEPGRAM_API_KEY: str
+    AZURE_API_KEY: str
+    AZURE_REGION: str
+    AZURE_LOCALES: str


 class AudioConfigUpdateForm(BaseModel):
@@ -169,6 +178,9 @@ async def get_audio_config(request: Request, user=Depends(get_admin_user)):
            "MODEL": request.app.state.config.STT_MODEL,
            "WHISPER_MODEL": request.app.state.config.WHISPER_MODEL,
            "DEEPGRAM_API_KEY": request.app.state.config.DEEPGRAM_API_KEY,
+            "AZURE_API_KEY": request.app.state.config.AUDIO_STT_AZURE_API_KEY,
+            "AZURE_REGION": request.app.state.config.AUDIO_STT_AZURE_REGION,
+            "AZURE_LOCALES": request.app.state.config.AUDIO_STT_AZURE_LOCALES,
        },
    }

@@ -195,6 +207,9 @@ async def update_audio_config(
    request.app.state.config.STT_MODEL = form_data.stt.MODEL
    request.app.state.config.WHISPER_MODEL = form_data.stt.WHISPER_MODEL
    request.app.state.config.DEEPGRAM_API_KEY = form_data.stt.DEEPGRAM_API_KEY
+    request.app.state.config.AUDIO_STT_AZURE_API_KEY = form_data.stt.AZURE_API_KEY
+    request.app.state.config.AUDIO_STT_AZURE_REGION = form_data.stt.AZURE_REGION
+    request.app.state.config.AUDIO_STT_AZURE_LOCALES = form_data.stt.AZURE_LOCALES

    if request.app.state.config.STT_ENGINE == "":
        request.app.state.faster_whisper_model = set_faster_whisper_model(
@@ -220,6 +235,9 @@ async def update_audio_config(
            "MODEL": request.app.state.config.STT_MODEL,
            "WHISPER_MODEL": request.app.state.config.WHISPER_MODEL,
            "DEEPGRAM_API_KEY": request.app.state.config.DEEPGRAM_API_KEY,
+            "AZURE_API_KEY": request.app.state.config.AUDIO_STT_AZURE_API_KEY,
+            "AZURE_REGION": request.app.state.config.AUDIO_STT_AZURE_REGION,
+            "AZURE_LOCALES": request.app.state.config.AUDIO_STT_AZURE_LOCALES,
        },
    }

@@ -496,10 +514,15 @@ def transcribe(request: Request, file_path):
        log.debug(data)
        return data
    elif request.app.state.config.STT_ENGINE == "openai":
-        if is_mp4_audio(file_path):
-            os.rename(file_path, file_path.replace(".wav", ".mp4"))
-            # Convert MP4 audio file to WAV format
-            convert_mp4_to_wav(file_path.replace(".wav", ".mp4"), file_path)
+        audio_format = get_audio_format(file_path)
+        if audio_format:
+            os.rename(file_path, file_path.replace(".wav", f".{audio_format}"))
+            # Convert unsupported audio file to WAV format
+            convert_audio_to_wav(
+                file_path.replace(".wav", f".{audio_format}"),
+                file_path,
+                audio_format,
+            )

        r = None
        try:
@@ -598,6 +621,107 @@ def transcribe(request: Request, file_path):
                    detail = f"External: {e}"
            raise Exception(detail if detail else "Open WebUI: Server Connection Error")

+    elif request.app.state.config.STT_ENGINE == "azure":
+        # Check file exists and size
+        if not os.path.exists(file_path):
+            raise HTTPException(
+                status_code=400,
+                detail="Audio file not found"
+            )
+
+        # Check file size (Azure has a larger limit of 200MB)
+        file_size = os.path.getsize(file_path)
+        if file_size > AZURE_MAX_FILE_SIZE:
+            raise HTTPException(
+                status_code=400,
+                detail=f"File size exceeds Azure's limit of {AZURE_MAX_FILE_SIZE_MB}MB",
+            )
+
+        api_key = request.app.state.config.AUDIO_STT_AZURE_API_KEY
+        region = request.app.state.config.AUDIO_STT_AZURE_REGION
+        locales = request.app.state.config.AUDIO_STT_AZURE_LOCALES
+
+        # IF NO LOCALES, USE DEFAULTS
+        if len(locales) < 2:
+            locales = ['en-US', 'es-ES', 'es-MX', 'fr-FR', 'hi-IN', 
+                       'it-IT','de-DE', 'en-GB', 'en-IN', 'ja-JP', 
+                       'ko-KR', 'pt-BR', 'zh-CN']
+            locales = ','.join(locales)
+
+
+        if not api_key or not region:
+            raise HTTPException(
+                status_code=400,
+                detail="Azure API key and region are required for Azure STT",
+            )
+
+        r = None
+        try:
+            # Prepare the request
+            data = {'definition': json.dumps({
+                                                "locales": locales.split(','),
+                                                "diarization": {"maxSpeakers": 3,"enabled": True}
+                                              } if locales else {}
+                                              )
+            }
+            url = f"https://{region}.api.cognitive.microsoft.com/speechtotext/transcriptions:transcribe?api-version=2024-11-15"
+            
+            # Use context manager to ensure file is properly closed
+            with open(file_path, 'rb') as audio_file:
+                r = requests.post(
+                    url=url,
+                    files={'audio': audio_file},
+                    data=data,
+                    headers={
+                        'Ocp-Apim-Subscription-Key': api_key,
+                    },
+                )
+
+            r.raise_for_status()
+            response = r.json()
+
+            # Extract transcript from response
+            if not response.get('combinedPhrases'):
+                raise ValueError("No transcription found in response")
+
+            # Get the full transcript from combinedPhrases
+            transcript = response['combinedPhrases'][0].get('text', '').strip()
+            if not transcript:
+                raise ValueError("Empty transcript in response")
+
+            data = {"text": transcript}
+
+            # Save transcript to json file (consistent with other providers)
+            transcript_file = f"{file_dir}/{id}.json"
+            with open(transcript_file, "w") as f:
+                json.dump(data, f)
+
+            log.debug(data)
+            return data
+
+        except (KeyError, IndexError, ValueError) as e:
+            log.exception("Error parsing Azure response")
+            raise HTTPException(
+                status_code=500,
+                detail=f"Failed to parse Azure response: {str(e)}",
+            )
+        except requests.exceptions.RequestException as e:
+            log.exception(e)
+            detail = None
+
+            try:
+                if r is not None and r.status_code != 200:
+                    res = r.json()
+                    if "error" in res:
+                        detail = f"External: {res['error'].get('message', '')}"
+            except Exception:
+                detail = f"External: {e}"
+
+            raise HTTPException(
+                status_code=getattr(r, 'status_code', 500) if r else 500,
+                detail=detail if detail else "Open WebUI: Server Connection Error",
+            )
+

 def compress_audio(file_path):
    if os.path.getsize(file_path) > MAX_FILE_SIZE:
@@ -231,11 +231,13 @@ async def ldap_auth(request: Request, response: Response, form_data: LdapForm):

        entry = connection_app.entries[0]
        username = str(entry[f"{LDAP_ATTRIBUTE_FOR_USERNAME}"]).lower()
-        email = str(entry[f"{LDAP_ATTRIBUTE_FOR_MAIL}"])
-        if not email or email == "" or email == "[]":
+        email = entry[f"{LDAP_ATTRIBUTE_FOR_MAIL}"]
+        if not email:
            raise HTTPException(400, "User does not have a valid email address.")
-        else:
+        elif isinstance(email, str):
            email = email.lower()
+        elif isinstance(email, list):
+            email = email[0].lower()

        cn = str(entry["cn"])
        user_dn = entry.entry_dn
@@ -455,6 +457,13 @@ async def signup(request: Request, response: Response, form_data: SignupForm):
            # Disable signup after the first user is created
            request.app.state.config.ENABLE_SIGNUP = False

+        # The password passed to bcrypt must be 72 bytes or fewer. If it is longer, it will be truncated before hashing.
+        if len(form_data.password.encode("utf-8")) > 72:
+            raise HTTPException(
+                status.HTTP_400_BAD_REQUEST,
+                detail=ERROR_MESSAGES.PASSWORD_TOO_LONG,
+            )
+
        hashed = get_password_hash(form_data.password)
        user = Auths.insert_new_auth(
            form_data.email.lower(),
@@ -1,6 +1,7 @@
 import logging
 import os
 import uuid
+from fnmatch import fnmatch
 from pathlib import Path
 from typing import Optional
 from urllib.parse import quote
@@ -177,6 +178,47 @@ async def list_files(user=Depends(get_verified_user), content: bool = Query(True
    return files


+############################
+# Search Files
+############################
+
+
+@router.get("/search", response_model=list[FileModelResponse])
+async def search_files(
+    filename: str = Query(
+        ...,
+        description="Filename pattern to search for. Supports wildcards such as '*.txt'",
+    ),
+    content: bool = Query(True),
+    user=Depends(get_verified_user),
+):
+    """
+    Search for files by filename with support for wildcard patterns.
+    """
+    # Get files according to user role
+    if user.role == "admin":
+        files = Files.get_files()
+    else:
+        files = Files.get_files_by_user_id(user.id)
+
+    # Get matching files
+    matching_files = [
+        file for file in files if fnmatch(file.filename.lower(), filename.lower())
+    ]
+
+    if not matching_files:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="No files found matching the pattern.",
+        )
+
+    if not content:
+        for file in matching_files:
+            del file.data["content"]
+
+    return matching_files
+
+
 ############################
 # Delete All Files
 ############################
@@ -159,6 +159,72 @@ async def create_new_knowledge(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail=ERROR_MESSAGES.FILE_EXISTS,
        )
+    
+
+
+############################
+# ReindexKnowledgeFiles
+############################
+
+
+@router.post("/reindex", response_model=bool)
+async def reindex_knowledge_files(
+    request: Request,
+    user=Depends(get_verified_user)
+):
+    if user.role != "admin":
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=ERROR_MESSAGES.UNAUTHORIZED,
+        )
+    
+    knowledge_bases = Knowledges.get_knowledge_bases()
+         
+    log.info(f"Starting reindexing for {len(knowledge_bases)} knowledge bases")
+    
+    for knowledge_base in knowledge_bases:
+        try:
+            files = Files.get_files_by_ids(knowledge_base.data.get("file_ids", []))
+
+            try:
+                if VECTOR_DB_CLIENT.has_collection(collection_name=knowledge_base.id):
+                    VECTOR_DB_CLIENT.delete_collection(
+                        collection_name=knowledge_base.id
+                    )
+            except Exception as e:
+                log.error(f"Error deleting collection {knowledge_base.id}: {str(e)}")
+                raise HTTPException(
+                    status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                    detail=f"Error deleting vector DB collection"
+                )
+            
+            failed_files = []
+            for file in files:
+                try:
+                    process_file(
+                        request,
+                        ProcessFileForm(file_id=file.id, collection_name=knowledge_base.id),
+                        user=user,
+                    )
+                except Exception as e:
+                    log.error(f"Error processing file {file.filename} (ID: {file.id}): {str(e)}")
+                    failed_files.append({"file_id": file.id, "error": str(e)})
+                    continue
+          
+        except Exception as e:
+            log.error(f"Error processing knowledge base {knowledge_base.id}: {str(e)}")
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Error processing knowledge base"
+            )
+        
+        if failed_files:
+            log.warning(f"Failed to process {len(failed_files)} files in knowledge base {knowledge_base.id}")
+            for failed in failed_files:
+                log.warning(f"File ID: {failed['file_id']}, Error: {failed['error']}")
+    
+    log.info("Reindexing completed successfully")
+    return True


 ############################
@@ -676,3 +742,6 @@ def add_files_to_knowledge_batch(
    return KnowledgeFilesResponse(
        **knowledge.model_dump(), files=Files.get_files_by_ids(existing_file_ids)
    )
+
+
+
@@ -60,6 +60,7 @@ from open_webui.retrieval.web.tavily import search_tavily
 from open_webui.retrieval.web.bing import search_bing
 from open_webui.retrieval.web.exa import search_exa
 from open_webui.retrieval.web.perplexity import search_perplexity
+from open_webui.retrieval.web.sougou import search_sougou

 from open_webui.retrieval.utils import (
    get_embedding_function,
@@ -411,6 +412,8 @@ async def get_rag_config(request: Request, user=Depends(get_admin_user)):
                "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
                "exa_api_key": request.app.state.config.EXA_API_KEY,
                "perplexity_api_key": request.app.state.config.PERPLEXITY_API_KEY,
+                "sougou_api_sid": request.app.state.config.SOUGOU_API_SID,
+                "sougou_api_sk": request.app.state.config.SOUGOU_API_SK,
                "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
                "trust_env": request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
                "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
@@ -478,6 +481,8 @@ class WebSearchConfig(BaseModel):
    bing_search_v7_subscription_key: Optional[str] = None
    exa_api_key: Optional[str] = None
    perplexity_api_key: Optional[str] = None
+    sougou_api_sid: Optional[str] = None
+    sougou_api_sk: Optional[str] = None
    result_count: Optional[int] = None
    concurrent_requests: Optional[int] = None
    trust_env: Optional[bool] = None
@@ -640,6 +645,12 @@ async def update_rag_config(
        request.app.state.config.PERPLEXITY_API_KEY = (
            form_data.web.search.perplexity_api_key
        )
+        request.app.state.config.SOUGOU_API_SID = (
+            form_data.web.search.sougou_api_sid
+        )
+        request.app.state.config.SOUGOU_API_SK = (
+            form_data.web.search.sougou_api_sk
+        )

        request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT = (
            form_data.web.search.result_count
@@ -712,6 +723,8 @@ async def update_rag_config(
                "bing_search_v7_subscription_key": request.app.state.config.BING_SEARCH_V7_SUBSCRIPTION_KEY,
                "exa_api_key": request.app.state.config.EXA_API_KEY,
                "perplexity_api_key": request.app.state.config.PERPLEXITY_API_KEY,
+                "sougou_api_sid": request.app.state.config.SOUGOU_API_SID,
+                "sougou_api_sk": request.app.state.config.SOUGOU_API_SK,
                "result_count": request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
                "concurrent_requests": request.app.state.config.RAG_WEB_SEARCH_CONCURRENT_REQUESTS,
                "trust_env": request.app.state.config.RAG_WEB_SEARCH_TRUST_ENV,
@@ -1267,6 +1280,7 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
    - TAVILY_API_KEY
    - EXA_API_KEY
    - PERPLEXITY_API_KEY
+    - SOUGOU_API_SID + SOUGOU_API_SK
    - SEARCHAPI_API_KEY + SEARCHAPI_ENGINE (by default `google`)
    - SERPAPI_API_KEY + SERPAPI_ENGINE (by default `google`)
    Args:
@@ -1438,6 +1452,17 @@ def search_web(request: Request, engine: str, query: str) -> list[SearchResult]:
            request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
            request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
        )
+    elif engine == 'sougou':
+        if request.app.state.config.SOUGOU_API_SID and request.app.state.config.SOUGOU_API_SK:
+            return search_sougou(
+                request.app.state.config.SOUGOU_API_SID,
+                request.app.state.config.SOUGOU_API_SK,
+                query,
+                request.app.state.config.RAG_WEB_SEARCH_RESULT_COUNT,
+                request.app.state.config.RAG_WEB_SEARCH_DOMAIN_FILTER_LIST,
+            )
+        else:
+            raise Exception("No SOUGOU_API_SID or SOUGOU_API_SK found in environment variables")
    else:
        raise Exception("No search engine API key found in environment variables")

@@ -10,11 +10,11 @@ from open_webui.models.tools import (
    ToolUserResponse,
    Tools,
 )
-from open_webui.utils.plugin import load_tools_module_by_id, replace_imports
+from open_webui.utils.plugin import load_tool_module_by_id, replace_imports
 from open_webui.config import CACHE_DIR
 from open_webui.constants import ERROR_MESSAGES
 from fastapi import APIRouter, Depends, HTTPException, Request, status
-from open_webui.utils.tools import get_tools_specs
+from open_webui.utils.tools import get_tool_specs
 from open_webui.utils.auth import get_admin_user, get_verified_user
 from open_webui.utils.access_control import has_access, has_permission
 from open_webui.env import SRC_LOG_LEVELS
@@ -45,7 +45,7 @@ async def get_tools(request: Request, user=Depends(get_verified_user)):
        )

    tools = Tools.get_tools()
-    for idx, server in enumerate(request.app.state.TOOL_SERVERS):
+    for server in request.app.state.TOOL_SERVERS:
        tools.append(
            ToolUserResponse(
                **{
@@ -60,7 +60,7 @@ async def get_tools(request: Request, user=Depends(get_verified_user)):
                        .get("description", ""),
                    },
                    "access_control": request.app.state.config.TOOL_SERVER_CONNECTIONS[
-                        idx
+                        server["idx"]
                    ]
                    .get("config", {})
                    .get("access_control", None),
@@ -137,15 +137,15 @@ async def create_new_tools(
    if tools is None:
        try:
            form_data.content = replace_imports(form_data.content)
-            tools_module, frontmatter = load_tools_module_by_id(
+            tool_module, frontmatter = load_tool_module_by_id(
                form_data.id, content=form_data.content
            )
            form_data.meta.manifest = frontmatter

            TOOLS = request.app.state.TOOLS
-            TOOLS[form_data.id] = tools_module
+            TOOLS[form_data.id] = tool_module

-            specs = get_tools_specs(TOOLS[form_data.id])
+            specs = get_tool_specs(TOOLS[form_data.id])
            tools = Tools.insert_new_tool(user.id, form_data, specs)

            tool_cache_dir = CACHE_DIR / "tools" / form_data.id
@@ -226,15 +226,13 @@ async def update_tools_by_id(

    try:
        form_data.content = replace_imports(form_data.content)
-        tools_module, frontmatter = load_tools_module_by_id(
-            id, content=form_data.content
-        )
+        tool_module, frontmatter = load_tool_module_by_id(id, content=form_data.content)
        form_data.meta.manifest = frontmatter

        TOOLS = request.app.state.TOOLS
-        TOOLS[id] = tools_module
+        TOOLS[id] = tool_module

-        specs = get_tools_specs(TOOLS[id])
+        specs = get_tool_specs(TOOLS[id])

        updated = {
            **form_data.model_dump(exclude={"id"}),
@@ -332,7 +330,7 @@ async def get_tools_valves_spec_by_id(
        if id in request.app.state.TOOLS:
            tools_module = request.app.state.TOOLS[id]
        else:
-            tools_module, _ = load_tools_module_by_id(id)
+            tools_module, _ = load_tool_module_by_id(id)
            request.app.state.TOOLS[id] = tools_module

        if hasattr(tools_module, "Valves"):
@@ -375,7 +373,7 @@ async def update_tools_valves_by_id(
    if id in request.app.state.TOOLS:
        tools_module = request.app.state.TOOLS[id]
    else:
-        tools_module, _ = load_tools_module_by_id(id)
+        tools_module, _ = load_tool_module_by_id(id)
        request.app.state.TOOLS[id] = tools_module

    if not hasattr(tools_module, "Valves"):
@@ -431,7 +429,7 @@ async def get_tools_user_valves_spec_by_id(
        if id in request.app.state.TOOLS:
            tools_module = request.app.state.TOOLS[id]
        else:
-            tools_module, _ = load_tools_module_by_id(id)
+            tools_module, _ = load_tool_module_by_id(id)
            request.app.state.TOOLS[id] = tools_module

        if hasattr(tools_module, "UserValves"):
@@ -455,7 +453,7 @@ async def update_tools_user_valves_by_id(
        if id in request.app.state.TOOLS:
            tools_module = request.app.state.TOOLS[id]
        else:
-            tools_module, _ = load_tools_module_by_id(id)
+            tools_module, _ = load_tool_module_by_id(id)
            request.app.state.TOOLS[id] = tools_module

        if hasattr(tools_module, "UserValves"):