diff --git a/backend/open_webui/retrieval/vector/dbs/pgvector.py b/backend/open_webui/retrieval/vector/dbs/pgvector.py index 4775ff21f..90e65b9ad 100644 --- a/backend/open_webui/retrieval/vector/dbs/pgvector.py +++ b/backend/open_webui/retrieval/vector/dbs/pgvector.py @@ -34,6 +34,7 @@ from open_webui.retrieval.vector.main import ( SearchResult, GetResult, ) +from open_webui.utils.misc import sanitize_text_for_db from open_webui.config import ( PGVECTOR_DB_URL, PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH, @@ -289,7 +290,9 @@ class PgvectorClient(VectorDBBase): vector = self.adjust_vector_length(item['vector']) # Use raw SQL for BYTEA/pgcrypto # Ensure metadata is converted to its JSON text representation - json_metadata = json.dumps(item['metadata']) + # Sanitize to strip null bytes / surrogates that PostgreSQL cannot store + json_metadata = sanitize_text_for_db(json.dumps(item['metadata'])) + item_text = sanitize_text_for_db(item['text']) self.session.execute( text(""" INSERT INTO document_chunk @@ -305,7 +308,7 @@ class PgvectorClient(VectorDBBase): 'id': item['id'], 'vector': vector, 'collection_name': collection_name, - 'text': item['text'], + 'text': item_text, 'metadata_text': json_metadata, 'key': PGVECTOR_PGCRYPTO_KEY, }, @@ -338,7 +341,9 @@ class PgvectorClient(VectorDBBase): if PGVECTOR_PGCRYPTO: for item in items: vector = self.adjust_vector_length(item['vector']) - json_metadata = json.dumps(item['metadata']) + # Sanitize to strip null bytes / surrogates that PostgreSQL cannot store + json_metadata = sanitize_text_for_db(json.dumps(item['metadata'])) + item_text = sanitize_text_for_db(item['text']) self.session.execute( text(""" INSERT INTO document_chunk @@ -358,7 +363,7 @@ class PgvectorClient(VectorDBBase): 'id': item['id'], 'vector': vector, 'collection_name': collection_name, - 'text': item['text'], + 'text': item_text, 'metadata_text': json_metadata, 'key': PGVECTOR_PGCRYPTO_KEY, }, diff --git a/backend/open_webui/retrieval/vector/utils.py b/backend/open_webui/retrieval/vector/utils.py index b2e2fed76..3ee413eaf 100644 --- a/backend/open_webui/retrieval/vector/utils.py +++ b/backend/open_webui/retrieval/vector/utils.py @@ -1,5 +1,7 @@ from datetime import datetime +from open_webui.utils.misc import sanitize_text_for_db + KEYS_TO_EXCLUDE = ['content', 'pages', 'tables', 'paragraphs', 'sections', 'figures'] @@ -12,7 +14,8 @@ def filter_metadata(metadata: dict[str, any]) -> dict[str, any]: def process_metadata( metadata: dict[str, any], ) -> dict[str, any]: - # Removes large fields and converts non-serializable types (datetime, list, dict) to strings. + # Removes large fields, converts non-serializable types (datetime, list, dict) to strings, + # and sanitizes strings for database storage (strips null bytes and invalid surrogates). result = {} for key, value in metadata.items(): # Skip large fields @@ -20,7 +23,8 @@ def process_metadata( continue # Convert non-serializable fields to strings if isinstance(value, (datetime, list, dict)): - result[key] = str(value) + result[key] = sanitize_text_for_db(str(value)) else: - result[key] = value + result[key] = sanitize_text_for_db(value) return result +