Merge branch 'dev' into vector-search-branch
This commit is contained in:
hiwylee
@@ -11,6 +11,8 @@ from open_webui.retrieval.vector.main import (
|
||||
SearchResult,
|
||||
GetResult,
|
||||
)
|
||||
from open_webui.retrieval.vector.utils import stringify_metadata
|
||||
|
||||
from open_webui.config import (
|
||||
CHROMA_DATA_PATH,
|
||||
CHROMA_HTTP_HOST,
|
||||
@@ -144,7 +146,7 @@ class ChromaClient(VectorDBBase):
|
||||
ids = [item["id"] for item in items]
|
||||
documents = [item["text"] for item in items]
|
||||
embeddings = [item["vector"] for item in items]
|
||||
metadatas = [item["metadata"] for item in items]
|
||||
metadatas = [stringify_metadata(item["metadata"]) for item in items]
|
||||
|
||||
for batch in create_batches(
|
||||
api=self.client,
|
||||
@@ -164,7 +166,7 @@ class ChromaClient(VectorDBBase):
|
||||
ids = [item["id"] for item in items]
|
||||
documents = [item["text"] for item in items]
|
||||
embeddings = [item["vector"] for item in items]
|
||||
metadatas = [item["metadata"] for item in items]
|
||||
metadatas = [stringify_metadata(item["metadata"]) for item in items]
|
||||
|
||||
collection.upsert(
|
||||
ids=ids, documents=documents, embeddings=embeddings, metadatas=metadatas
|
||||
|
||||
@@ -3,6 +3,8 @@ from pymilvus import FieldSchema, DataType
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from open_webui.retrieval.vector.utils import stringify_metadata
|
||||
from open_webui.retrieval.vector.main import (
|
||||
VectorDBBase,
|
||||
VectorItem,
|
||||
@@ -311,7 +313,7 @@ class MilvusClient(VectorDBBase):
|
||||
"id": item["id"],
|
||||
"vector": item["vector"],
|
||||
"data": {"text": item["text"]},
|
||||
"metadata": item["metadata"],
|
||||
"metadata": stringify_metadata(item["metadata"]),
|
||||
}
|
||||
for item in items
|
||||
],
|
||||
@@ -347,7 +349,7 @@ class MilvusClient(VectorDBBase):
|
||||
"id": item["id"],
|
||||
"vector": item["vector"],
|
||||
"data": {"text": item["text"]},
|
||||
"metadata": item["metadata"],
|
||||
"metadata": stringify_metadata(item["metadata"]),
|
||||
}
|
||||
for item in items
|
||||
],
|
||||
|
||||
@@ -157,10 +157,10 @@ class OpenSearchClient(VectorDBBase):
|
||||
|
||||
for field, value in filter.items():
|
||||
query_body["query"]["bool"]["filter"].append(
|
||||
{"match": {"metadata." + str(field): value}}
|
||||
{"term": {"metadata." + str(field) + ".keyword": value}}
|
||||
)
|
||||
|
||||
size = limit if limit else 10
|
||||
size = limit if limit else 10000
|
||||
|
||||
try:
|
||||
result = self.client.search(
|
||||
@@ -206,6 +206,7 @@ class OpenSearchClient(VectorDBBase):
|
||||
for item in batch
|
||||
]
|
||||
bulk(self.client, actions)
|
||||
self.client.indices.refresh(self._get_index_name(collection_name))
|
||||
|
||||
def upsert(self, collection_name: str, items: list[VectorItem]):
|
||||
self._create_index_if_not_exists(
|
||||
@@ -228,6 +229,7 @@ class OpenSearchClient(VectorDBBase):
|
||||
for item in batch
|
||||
]
|
||||
bulk(self.client, actions)
|
||||
self.client.indices.refresh(self._get_index_name(collection_name))
|
||||
|
||||
def delete(
|
||||
self,
|
||||
@@ -251,11 +253,12 @@ class OpenSearchClient(VectorDBBase):
|
||||
}
|
||||
for field, value in filter.items():
|
||||
query_body["query"]["bool"]["filter"].append(
|
||||
{"match": {"metadata." + str(field): value}}
|
||||
{"term": {"metadata." + str(field) + ".keyword": value}}
|
||||
)
|
||||
self.client.delete_by_query(
|
||||
index=self._get_index_name(collection_name), body=query_body
|
||||
)
|
||||
self.client.indices.refresh(self._get_index_name(collection_name))
|
||||
|
||||
def reset(self):
|
||||
indices = self.client.indices.get(index=f"{self.index_prefix}_*")
|
||||
|
||||
@@ -18,7 +18,7 @@ from sqlalchemy import (
|
||||
values,
|
||||
)
|
||||
from sqlalchemy.sql import true
|
||||
from sqlalchemy.pool import NullPool
|
||||
from sqlalchemy.pool import NullPool, QueuePool
|
||||
|
||||
from sqlalchemy.orm import declarative_base, scoped_session, sessionmaker
|
||||
from sqlalchemy.dialects.postgresql import JSONB, array
|
||||
@@ -26,6 +26,8 @@ from pgvector.sqlalchemy import Vector
|
||||
from sqlalchemy.ext.mutable import MutableDict
|
||||
from sqlalchemy.exc import NoSuchTableError
|
||||
|
||||
|
||||
from open_webui.retrieval.vector.utils import stringify_metadata
|
||||
from open_webui.retrieval.vector.main import (
|
||||
VectorDBBase,
|
||||
VectorItem,
|
||||
@@ -37,6 +39,10 @@ from open_webui.config import (
|
||||
PGVECTOR_INITIALIZE_MAX_VECTOR_LENGTH,
|
||||
PGVECTOR_PGCRYPTO,
|
||||
PGVECTOR_PGCRYPTO_KEY,
|
||||
PGVECTOR_POOL_SIZE,
|
||||
PGVECTOR_POOL_MAX_OVERFLOW,
|
||||
PGVECTOR_POOL_TIMEOUT,
|
||||
PGVECTOR_POOL_RECYCLE,
|
||||
)
|
||||
|
||||
from open_webui.env import SRC_LOG_LEVELS
|
||||
@@ -80,9 +86,24 @@ class PgvectorClient(VectorDBBase):
|
||||
|
||||
self.session = Session
|
||||
else:
|
||||
engine = create_engine(
|
||||
PGVECTOR_DB_URL, pool_pre_ping=True, poolclass=NullPool
|
||||
)
|
||||
if isinstance(PGVECTOR_POOL_SIZE, int):
|
||||
if PGVECTOR_POOL_SIZE > 0:
|
||||
engine = create_engine(
|
||||
PGVECTOR_DB_URL,
|
||||
pool_size=PGVECTOR_POOL_SIZE,
|
||||
max_overflow=PGVECTOR_POOL_MAX_OVERFLOW,
|
||||
pool_timeout=PGVECTOR_POOL_TIMEOUT,
|
||||
pool_recycle=PGVECTOR_POOL_RECYCLE,
|
||||
pool_pre_ping=True,
|
||||
poolclass=QueuePool,
|
||||
)
|
||||
else:
|
||||
engine = create_engine(
|
||||
PGVECTOR_DB_URL, pool_pre_ping=True, poolclass=NullPool
|
||||
)
|
||||
else:
|
||||
engine = create_engine(PGVECTOR_DB_URL, pool_pre_ping=True)
|
||||
|
||||
SessionLocal = sessionmaker(
|
||||
autocommit=False, autoflush=False, bind=engine, expire_on_commit=False
|
||||
)
|
||||
@@ -216,7 +237,7 @@ class PgvectorClient(VectorDBBase):
|
||||
vector=vector,
|
||||
collection_name=collection_name,
|
||||
text=item["text"],
|
||||
vmetadata=item["metadata"],
|
||||
vmetadata=stringify_metadata(item["metadata"]),
|
||||
)
|
||||
new_items.append(new_chunk)
|
||||
self.session.bulk_save_objects(new_items)
|
||||
@@ -273,7 +294,7 @@ class PgvectorClient(VectorDBBase):
|
||||
if existing:
|
||||
existing.vector = vector
|
||||
existing.text = item["text"]
|
||||
existing.vmetadata = item["metadata"]
|
||||
existing.vmetadata = stringify_metadata(item["metadata"])
|
||||
existing.collection_name = (
|
||||
collection_name # Update collection_name if necessary
|
||||
)
|
||||
@@ -283,7 +304,7 @@ class PgvectorClient(VectorDBBase):
|
||||
vector=vector,
|
||||
collection_name=collection_name,
|
||||
text=item["text"],
|
||||
vmetadata=item["metadata"],
|
||||
vmetadata=stringify_metadata(item["metadata"]),
|
||||
)
|
||||
self.session.add(new_chunk)
|
||||
self.session.commit()
|
||||
|
||||
@@ -18,6 +18,7 @@ from open_webui.config import (
|
||||
QDRANT_ON_DISK,
|
||||
QDRANT_GRPC_PORT,
|
||||
QDRANT_PREFER_GRPC,
|
||||
QDRANT_COLLECTION_PREFIX,
|
||||
)
|
||||
from open_webui.env import SRC_LOG_LEVELS
|
||||
|
||||
@@ -29,7 +30,7 @@ log.setLevel(SRC_LOG_LEVELS["RAG"])
|
||||
|
||||
class QdrantClient(VectorDBBase):
|
||||
def __init__(self):
|
||||
self.collection_prefix = "open-webui"
|
||||
self.collection_prefix = QDRANT_COLLECTION_PREFIX
|
||||
self.QDRANT_URI = QDRANT_URI
|
||||
self.QDRANT_API_KEY = QDRANT_API_KEY
|
||||
self.QDRANT_ON_DISK = QDRANT_ON_DISK
|
||||
@@ -86,6 +87,25 @@ class QdrantClient(VectorDBBase):
|
||||
),
|
||||
)
|
||||
|
||||
# Create payload indexes for efficient filtering
|
||||
self.client.create_payload_index(
|
||||
collection_name=collection_name_with_prefix,
|
||||
field_name="metadata.hash",
|
||||
field_schema=models.KeywordIndexParams(
|
||||
type=models.KeywordIndexType.KEYWORD,
|
||||
is_tenant=False,
|
||||
on_disk=self.QDRANT_ON_DISK,
|
||||
),
|
||||
)
|
||||
self.client.create_payload_index(
|
||||
collection_name=collection_name_with_prefix,
|
||||
field_name="metadata.file_id",
|
||||
field_schema=models.KeywordIndexParams(
|
||||
type=models.KeywordIndexType.KEYWORD,
|
||||
is_tenant=False,
|
||||
on_disk=self.QDRANT_ON_DISK,
|
||||
),
|
||||
)
|
||||
log.info(f"collection {collection_name_with_prefix} successfully created!")
|
||||
|
||||
def _create_collection_if_not_exists(self, collection_name, dimension):
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import logging
|
||||
from typing import Optional, Tuple
|
||||
from typing import Optional, Tuple, List, Dict, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import grpc
|
||||
@@ -9,6 +9,7 @@ from open_webui.config import (
|
||||
QDRANT_ON_DISK,
|
||||
QDRANT_PREFER_GRPC,
|
||||
QDRANT_URI,
|
||||
QDRANT_COLLECTION_PREFIX,
|
||||
)
|
||||
from open_webui.env import SRC_LOG_LEVELS
|
||||
from open_webui.retrieval.vector.main import (
|
||||
@@ -23,14 +24,28 @@ from qdrant_client.http.models import PointStruct
|
||||
from qdrant_client.models import models
|
||||
|
||||
NO_LIMIT = 999999999
|
||||
TENANT_ID_FIELD = "tenant_id"
|
||||
DEFAULT_DIMENSION = 384
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log.setLevel(SRC_LOG_LEVELS["RAG"])
|
||||
|
||||
|
||||
def _tenant_filter(tenant_id: str) -> models.FieldCondition:
|
||||
return models.FieldCondition(
|
||||
key=TENANT_ID_FIELD, match=models.MatchValue(value=tenant_id)
|
||||
)
|
||||
|
||||
|
||||
def _metadata_filter(key: str, value: Any) -> models.FieldCondition:
|
||||
return models.FieldCondition(
|
||||
key=f"metadata.{key}", match=models.MatchValue(value=value)
|
||||
)
|
||||
|
||||
|
||||
class QdrantClient(VectorDBBase):
|
||||
def __init__(self):
|
||||
self.collection_prefix = "open-webui"
|
||||
self.collection_prefix = QDRANT_COLLECTION_PREFIX
|
||||
self.QDRANT_URI = QDRANT_URI
|
||||
self.QDRANT_API_KEY = QDRANT_API_KEY
|
||||
self.QDRANT_ON_DISK = QDRANT_ON_DISK
|
||||
@@ -38,24 +53,26 @@ class QdrantClient(VectorDBBase):
|
||||
self.GRPC_PORT = QDRANT_GRPC_PORT
|
||||
|
||||
if not self.QDRANT_URI:
|
||||
self.client = None
|
||||
return
|
||||
raise ValueError(
|
||||
"QDRANT_URI is not set. Please configure it in the environment variables."
|
||||
)
|
||||
|
||||
# Unified handling for either scheme
|
||||
parsed = urlparse(self.QDRANT_URI)
|
||||
host = parsed.hostname or self.QDRANT_URI
|
||||
http_port = parsed.port or 6333 # default REST port
|
||||
|
||||
if self.PREFER_GRPC:
|
||||
self.client = Qclient(
|
||||
self.client = (
|
||||
Qclient(
|
||||
host=host,
|
||||
port=http_port,
|
||||
grpc_port=self.GRPC_PORT,
|
||||
prefer_grpc=self.PREFER_GRPC,
|
||||
api_key=self.QDRANT_API_KEY,
|
||||
)
|
||||
else:
|
||||
self.client = Qclient(url=self.QDRANT_URI, api_key=self.QDRANT_API_KEY)
|
||||
if self.PREFER_GRPC
|
||||
else Qclient(url=self.QDRANT_URI, api_key=self.QDRANT_API_KEY)
|
||||
)
|
||||
|
||||
# Main collection types for multi-tenancy
|
||||
self.MEMORY_COLLECTION = f"{self.collection_prefix}_memories"
|
||||
@@ -65,23 +82,13 @@ class QdrantClient(VectorDBBase):
|
||||
self.HASH_BASED_COLLECTION = f"{self.collection_prefix}_hash-based"
|
||||
|
||||
def _result_to_get_result(self, points) -> GetResult:
|
||||
ids = []
|
||||
documents = []
|
||||
metadatas = []
|
||||
|
||||
ids, documents, metadatas = [], [], []
|
||||
for point in points:
|
||||
payload = point.payload
|
||||
ids.append(point.id)
|
||||
documents.append(payload["text"])
|
||||
metadatas.append(payload["metadata"])
|
||||
|
||||
return GetResult(
|
||||
**{
|
||||
"ids": [ids],
|
||||
"documents": [documents],
|
||||
"metadatas": [metadatas],
|
||||
}
|
||||
)
|
||||
return GetResult(ids=[ids], documents=[documents], metadatas=[metadatas])
|
||||
|
||||
def _get_collection_and_tenant_id(self, collection_name: str) -> Tuple[str, str]:
|
||||
"""
|
||||
@@ -113,143 +120,47 @@ class QdrantClient(VectorDBBase):
|
||||
else:
|
||||
return self.KNOWLEDGE_COLLECTION, tenant_id
|
||||
|
||||
def _extract_error_message(self, exception):
|
||||
"""
|
||||
Extract error message from either HTTP or gRPC exceptions
|
||||
|
||||
Returns:
|
||||
tuple: (status_code, error_message)
|
||||
"""
|
||||
# Check if it's an HTTP exception
|
||||
if isinstance(exception, UnexpectedResponse):
|
||||
try:
|
||||
error_data = exception.structured()
|
||||
error_msg = error_data.get("status", {}).get("error", "")
|
||||
return exception.status_code, error_msg
|
||||
except Exception as inner_e:
|
||||
log.error(f"Failed to parse HTTP error: {inner_e}")
|
||||
return exception.status_code, str(exception)
|
||||
|
||||
# Check if it's a gRPC exception
|
||||
elif isinstance(exception, grpc.RpcError):
|
||||
# Extract status code from gRPC error
|
||||
status_code = None
|
||||
if hasattr(exception, "code") and callable(exception.code):
|
||||
status_code = exception.code().value[0]
|
||||
|
||||
# Extract error message
|
||||
error_msg = str(exception)
|
||||
if "details =" in error_msg:
|
||||
# Parse the details line which contains the actual error message
|
||||
try:
|
||||
details_line = [
|
||||
line.strip()
|
||||
for line in error_msg.split("\n")
|
||||
if "details =" in line
|
||||
][0]
|
||||
error_msg = details_line.split("details =")[1].strip(' "')
|
||||
except (IndexError, AttributeError):
|
||||
# Fall back to full message if parsing fails
|
||||
pass
|
||||
|
||||
return status_code, error_msg
|
||||
|
||||
# For any other type of exception
|
||||
return None, str(exception)
|
||||
|
||||
def _is_collection_not_found_error(self, exception):
|
||||
"""
|
||||
Check if the exception is due to collection not found, supporting both HTTP and gRPC
|
||||
"""
|
||||
status_code, error_msg = self._extract_error_message(exception)
|
||||
|
||||
# HTTP error (404)
|
||||
if (
|
||||
status_code == 404
|
||||
and "Collection" in error_msg
|
||||
and "doesn't exist" in error_msg
|
||||
):
|
||||
return True
|
||||
|
||||
# gRPC error (NOT_FOUND status)
|
||||
if (
|
||||
isinstance(exception, grpc.RpcError)
|
||||
and exception.code() == grpc.StatusCode.NOT_FOUND
|
||||
):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _is_dimension_mismatch_error(self, exception):
|
||||
"""
|
||||
Check if the exception is due to dimension mismatch, supporting both HTTP and gRPC
|
||||
"""
|
||||
status_code, error_msg = self._extract_error_message(exception)
|
||||
|
||||
# Common patterns in both HTTP and gRPC
|
||||
return (
|
||||
"Vector dimension error" in error_msg
|
||||
or "dimensions mismatch" in error_msg
|
||||
or "invalid vector size" in error_msg
|
||||
)
|
||||
|
||||
def _create_multi_tenant_collection_if_not_exists(
|
||||
self, mt_collection_name: str, dimension: int = 384
|
||||
def _create_multi_tenant_collection(
|
||||
self, mt_collection_name: str, dimension: int = DEFAULT_DIMENSION
|
||||
):
|
||||
"""
|
||||
Creates a collection with multi-tenancy configuration if it doesn't exist.
|
||||
Default dimension is set to 384 which corresponds to 'sentence-transformers/all-MiniLM-L6-v2'.
|
||||
When creating collections dynamically (insert/upsert), the actual vector dimensions will be used.
|
||||
Creates a collection with multi-tenancy configuration and payload indexes for tenant_id and metadata fields.
|
||||
"""
|
||||
try:
|
||||
# Try to create the collection directly - will fail if it already exists
|
||||
self.client.create_collection(
|
||||
collection_name=mt_collection_name,
|
||||
vectors_config=models.VectorParams(
|
||||
size=dimension,
|
||||
distance=models.Distance.COSINE,
|
||||
on_disk=self.QDRANT_ON_DISK,
|
||||
),
|
||||
hnsw_config=models.HnswConfigDiff(
|
||||
payload_m=16, # Enable per-tenant indexing
|
||||
m=0,
|
||||
on_disk=self.QDRANT_ON_DISK,
|
||||
),
|
||||
)
|
||||
self.client.create_collection(
|
||||
collection_name=mt_collection_name,
|
||||
vectors_config=models.VectorParams(
|
||||
size=dimension,
|
||||
distance=models.Distance.COSINE,
|
||||
on_disk=self.QDRANT_ON_DISK,
|
||||
),
|
||||
)
|
||||
log.info(
|
||||
f"Multi-tenant collection {mt_collection_name} created with dimension {dimension}!"
|
||||
)
|
||||
|
||||
# Create tenant ID payload index
|
||||
self.client.create_payload_index(
|
||||
collection_name=mt_collection_name,
|
||||
field_name=TENANT_ID_FIELD,
|
||||
field_schema=models.KeywordIndexParams(
|
||||
type=models.KeywordIndexType.KEYWORD,
|
||||
is_tenant=True,
|
||||
on_disk=self.QDRANT_ON_DISK,
|
||||
),
|
||||
)
|
||||
|
||||
for field in ("metadata.hash", "metadata.file_id"):
|
||||
self.client.create_payload_index(
|
||||
collection_name=mt_collection_name,
|
||||
field_name="tenant_id",
|
||||
field_name=field,
|
||||
field_schema=models.KeywordIndexParams(
|
||||
type=models.KeywordIndexType.KEYWORD,
|
||||
is_tenant=True,
|
||||
on_disk=self.QDRANT_ON_DISK,
|
||||
),
|
||||
wait=True,
|
||||
)
|
||||
|
||||
log.info(
|
||||
f"Multi-tenant collection {mt_collection_name} created with dimension {dimension}!"
|
||||
)
|
||||
except (UnexpectedResponse, grpc.RpcError) as e:
|
||||
# Check for the specific error indicating collection already exists
|
||||
status_code, error_msg = self._extract_error_message(e)
|
||||
|
||||
# HTTP status code 409 or gRPC ALREADY_EXISTS
|
||||
if (isinstance(e, UnexpectedResponse) and status_code == 409) or (
|
||||
isinstance(e, grpc.RpcError)
|
||||
and e.code() == grpc.StatusCode.ALREADY_EXISTS
|
||||
):
|
||||
if "already exists" in error_msg:
|
||||
log.debug(f"Collection {mt_collection_name} already exists")
|
||||
return
|
||||
# If it's not an already exists error, re-raise
|
||||
raise e
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
def _create_points(self, items: list[VectorItem], tenant_id: str):
|
||||
def _create_points(
|
||||
self, items: List[VectorItem], tenant_id: str
|
||||
) -> List[PointStruct]:
|
||||
"""
|
||||
Create point structs from vector items with tenant ID.
|
||||
"""
|
||||
@@ -260,56 +171,42 @@ class QdrantClient(VectorDBBase):
|
||||
payload={
|
||||
"text": item["text"],
|
||||
"metadata": item["metadata"],
|
||||
"tenant_id": tenant_id,
|
||||
TENANT_ID_FIELD: tenant_id,
|
||||
},
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
|
||||
def _ensure_collection(
|
||||
self, mt_collection_name: str, dimension: int = DEFAULT_DIMENSION
|
||||
):
|
||||
"""
|
||||
Ensure the collection exists and payload indexes are created for tenant_id and metadata fields.
|
||||
"""
|
||||
if not self.client.collection_exists(collection_name=mt_collection_name):
|
||||
self._create_multi_tenant_collection(mt_collection_name, dimension)
|
||||
|
||||
def has_collection(self, collection_name: str) -> bool:
|
||||
"""
|
||||
Check if a logical collection exists by checking for any points with the tenant ID.
|
||||
"""
|
||||
if not self.client:
|
||||
return False
|
||||
|
||||
# Map to multi-tenant collection and tenant ID
|
||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||
|
||||
# Create tenant filter
|
||||
tenant_filter = models.FieldCondition(
|
||||
key="tenant_id", match=models.MatchValue(value=tenant_id)
|
||||
)
|
||||
|
||||
try:
|
||||
# Try directly querying - most of the time collection should exist
|
||||
response = self.client.query_points(
|
||||
collection_name=mt_collection,
|
||||
query_filter=models.Filter(must=[tenant_filter]),
|
||||
limit=1,
|
||||
)
|
||||
|
||||
# Collection exists with this tenant ID if there are points
|
||||
return len(response.points) > 0
|
||||
except (UnexpectedResponse, grpc.RpcError) as e:
|
||||
if self._is_collection_not_found_error(e):
|
||||
log.debug(f"Collection {mt_collection} doesn't exist")
|
||||
return False
|
||||
else:
|
||||
# For other API errors, log and return False
|
||||
_, error_msg = self._extract_error_message(e)
|
||||
log.warning(f"Unexpected Qdrant error: {error_msg}")
|
||||
return False
|
||||
except Exception as e:
|
||||
# For any other errors, log and return False
|
||||
log.debug(f"Error checking collection {mt_collection}: {e}")
|
||||
if not self.client.collection_exists(collection_name=mt_collection):
|
||||
return False
|
||||
tenant_filter = _tenant_filter(tenant_id)
|
||||
count_result = self.client.count(
|
||||
collection_name=mt_collection,
|
||||
count_filter=models.Filter(must=[tenant_filter]),
|
||||
)
|
||||
return count_result.count > 0
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection_name: str,
|
||||
ids: Optional[list[str]] = None,
|
||||
filter: Optional[dict] = None,
|
||||
ids: Optional[List[str]] = None,
|
||||
filter: Optional[Dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Delete vectors by ID or filter from a collection with tenant isolation.
|
||||
@@ -317,189 +214,76 @@ class QdrantClient(VectorDBBase):
|
||||
if not self.client:
|
||||
return None
|
||||
|
||||
# Map to multi-tenant collection and tenant ID
|
||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||
if not self.client.collection_exists(collection_name=mt_collection):
|
||||
log.debug(f"Collection {mt_collection} doesn't exist, nothing to delete")
|
||||
return None
|
||||
|
||||
# Create tenant filter
|
||||
tenant_filter = models.FieldCondition(
|
||||
key="tenant_id", match=models.MatchValue(value=tenant_id)
|
||||
must_conditions = [_tenant_filter(tenant_id)]
|
||||
should_conditions = []
|
||||
if ids:
|
||||
should_conditions = [_metadata_filter("id", id_value) for id_value in ids]
|
||||
elif filter:
|
||||
must_conditions += [_metadata_filter(k, v) for k, v in filter.items()]
|
||||
|
||||
return self.client.delete(
|
||||
collection_name=mt_collection,
|
||||
points_selector=models.FilterSelector(
|
||||
filter=models.Filter(must=must_conditions, should=should_conditions)
|
||||
),
|
||||
)
|
||||
|
||||
must_conditions = [tenant_filter]
|
||||
should_conditions = []
|
||||
|
||||
if ids:
|
||||
for id_value in ids:
|
||||
should_conditions.append(
|
||||
models.FieldCondition(
|
||||
key="metadata.id",
|
||||
match=models.MatchValue(value=id_value),
|
||||
),
|
||||
)
|
||||
elif filter:
|
||||
for key, value in filter.items():
|
||||
must_conditions.append(
|
||||
models.FieldCondition(
|
||||
key=f"metadata.{key}",
|
||||
match=models.MatchValue(value=value),
|
||||
),
|
||||
)
|
||||
|
||||
try:
|
||||
# Try to delete directly - most of the time collection should exist
|
||||
update_result = self.client.delete(
|
||||
collection_name=mt_collection,
|
||||
points_selector=models.FilterSelector(
|
||||
filter=models.Filter(must=must_conditions, should=should_conditions)
|
||||
),
|
||||
)
|
||||
|
||||
return update_result
|
||||
except (UnexpectedResponse, grpc.RpcError) as e:
|
||||
if self._is_collection_not_found_error(e):
|
||||
log.debug(
|
||||
f"Collection {mt_collection} doesn't exist, nothing to delete"
|
||||
)
|
||||
return None
|
||||
else:
|
||||
# For other API errors, log and re-raise
|
||||
_, error_msg = self._extract_error_message(e)
|
||||
log.warning(f"Unexpected Qdrant error: {error_msg}")
|
||||
raise
|
||||
except Exception as e:
|
||||
# For non-Qdrant exceptions, re-raise
|
||||
raise
|
||||
|
||||
def search(
|
||||
self, collection_name: str, vectors: list[list[float | int]], limit: int
|
||||
self, collection_name: str, vectors: List[List[float | int]], limit: int
|
||||
) -> Optional[SearchResult]:
|
||||
"""
|
||||
Search for the nearest neighbor items based on the vectors with tenant isolation.
|
||||
"""
|
||||
if not self.client:
|
||||
if not self.client or not vectors:
|
||||
return None
|
||||
|
||||
# Map to multi-tenant collection and tenant ID
|
||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||
|
||||
# Get the vector dimension from the query vector
|
||||
dimension = len(vectors[0]) if vectors and len(vectors) > 0 else None
|
||||
|
||||
try:
|
||||
# Try the search operation directly - most of the time collection should exist
|
||||
|
||||
# Create tenant filter
|
||||
tenant_filter = models.FieldCondition(
|
||||
key="tenant_id", match=models.MatchValue(value=tenant_id)
|
||||
)
|
||||
|
||||
# Ensure vector dimensions match the collection
|
||||
collection_dim = self.client.get_collection(
|
||||
mt_collection
|
||||
).config.params.vectors.size
|
||||
|
||||
if collection_dim != dimension:
|
||||
if collection_dim < dimension:
|
||||
vectors = [vector[:collection_dim] for vector in vectors]
|
||||
else:
|
||||
vectors = [
|
||||
vector + [0] * (collection_dim - dimension)
|
||||
for vector in vectors
|
||||
]
|
||||
|
||||
# Search with tenant filter
|
||||
prefetch_query = models.Prefetch(
|
||||
filter=models.Filter(must=[tenant_filter]),
|
||||
limit=NO_LIMIT,
|
||||
)
|
||||
query_response = self.client.query_points(
|
||||
collection_name=mt_collection,
|
||||
query=vectors[0],
|
||||
prefetch=prefetch_query,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
get_result = self._result_to_get_result(query_response.points)
|
||||
return SearchResult(
|
||||
ids=get_result.ids,
|
||||
documents=get_result.documents,
|
||||
metadatas=get_result.metadatas,
|
||||
# qdrant distance is [-1, 1], normalize to [0, 1]
|
||||
distances=[
|
||||
[(point.score + 1.0) / 2.0 for point in query_response.points]
|
||||
],
|
||||
)
|
||||
except (UnexpectedResponse, grpc.RpcError) as e:
|
||||
if self._is_collection_not_found_error(e):
|
||||
log.debug(
|
||||
f"Collection {mt_collection} doesn't exist, search returns None"
|
||||
)
|
||||
return None
|
||||
else:
|
||||
# For other API errors, log and re-raise
|
||||
_, error_msg = self._extract_error_message(e)
|
||||
log.warning(f"Unexpected Qdrant error during search: {error_msg}")
|
||||
raise
|
||||
except Exception as e:
|
||||
# For non-Qdrant exceptions, log and return None
|
||||
log.exception(f"Error searching collection '{collection_name}': {e}")
|
||||
if not self.client.collection_exists(collection_name=mt_collection):
|
||||
log.debug(f"Collection {mt_collection} doesn't exist, search returns None")
|
||||
return None
|
||||
|
||||
def query(self, collection_name: str, filter: dict, limit: Optional[int] = None):
|
||||
tenant_filter = _tenant_filter(tenant_id)
|
||||
query_response = self.client.query_points(
|
||||
collection_name=mt_collection,
|
||||
query=vectors[0],
|
||||
limit=limit,
|
||||
query_filter=models.Filter(must=[tenant_filter]),
|
||||
)
|
||||
get_result = self._result_to_get_result(query_response.points)
|
||||
return SearchResult(
|
||||
ids=get_result.ids,
|
||||
documents=get_result.documents,
|
||||
metadatas=get_result.metadatas,
|
||||
distances=[[(point.score + 1.0) / 2.0 for point in query_response.points]],
|
||||
)
|
||||
|
||||
def query(
|
||||
self, collection_name: str, filter: Dict[str, Any], limit: Optional[int] = None
|
||||
):
|
||||
"""
|
||||
Query points with filters and tenant isolation.
|
||||
"""
|
||||
if not self.client:
|
||||
return None
|
||||
|
||||
# Map to multi-tenant collection and tenant ID
|
||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||
|
||||
# Set default limit if not provided
|
||||
if not self.client.collection_exists(collection_name=mt_collection):
|
||||
log.debug(f"Collection {mt_collection} doesn't exist, query returns None")
|
||||
return None
|
||||
if limit is None:
|
||||
limit = NO_LIMIT
|
||||
|
||||
# Create tenant filter
|
||||
tenant_filter = models.FieldCondition(
|
||||
key="tenant_id", match=models.MatchValue(value=tenant_id)
|
||||
)
|
||||
|
||||
# Create metadata filters
|
||||
field_conditions = []
|
||||
for key, value in filter.items():
|
||||
field_conditions.append(
|
||||
models.FieldCondition(
|
||||
key=f"metadata.{key}", match=models.MatchValue(value=value)
|
||||
)
|
||||
)
|
||||
|
||||
# Combine tenant filter with metadata filters
|
||||
tenant_filter = _tenant_filter(tenant_id)
|
||||
field_conditions = [_metadata_filter(k, v) for k, v in filter.items()]
|
||||
combined_filter = models.Filter(must=[tenant_filter, *field_conditions])
|
||||
|
||||
try:
|
||||
# Try the query directly - most of the time collection should exist
|
||||
points = self.client.query_points(
|
||||
collection_name=mt_collection,
|
||||
query_filter=combined_filter,
|
||||
limit=limit,
|
||||
)
|
||||
|
||||
return self._result_to_get_result(points.points)
|
||||
except (UnexpectedResponse, grpc.RpcError) as e:
|
||||
if self._is_collection_not_found_error(e):
|
||||
log.debug(
|
||||
f"Collection {mt_collection} doesn't exist, query returns None"
|
||||
)
|
||||
return None
|
||||
else:
|
||||
# For other API errors, log and re-raise
|
||||
_, error_msg = self._extract_error_message(e)
|
||||
log.warning(f"Unexpected Qdrant error during query: {error_msg}")
|
||||
raise
|
||||
except Exception as e:
|
||||
# For non-Qdrant exceptions, log and re-raise
|
||||
log.exception(f"Error querying collection '{collection_name}': {e}")
|
||||
return None
|
||||
points = self.client.query_points(
|
||||
collection_name=mt_collection,
|
||||
query_filter=combined_filter,
|
||||
limit=limit,
|
||||
)
|
||||
return self._result_to_get_result(points.points)
|
||||
|
||||
def get(self, collection_name: str) -> Optional[GetResult]:
|
||||
"""
|
||||
@@ -507,169 +291,36 @@ class QdrantClient(VectorDBBase):
|
||||
"""
|
||||
if not self.client:
|
||||
return None
|
||||
|
||||
# Map to multi-tenant collection and tenant ID
|
||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||
|
||||
# Create tenant filter
|
||||
tenant_filter = models.FieldCondition(
|
||||
key="tenant_id", match=models.MatchValue(value=tenant_id)
|
||||
)
|
||||
|
||||
try:
|
||||
# Try to get points directly - most of the time collection should exist
|
||||
points = self.client.query_points(
|
||||
collection_name=mt_collection,
|
||||
query_filter=models.Filter(must=[tenant_filter]),
|
||||
limit=NO_LIMIT,
|
||||
)
|
||||
|
||||
return self._result_to_get_result(points.points)
|
||||
except (UnexpectedResponse, grpc.RpcError) as e:
|
||||
if self._is_collection_not_found_error(e):
|
||||
log.debug(f"Collection {mt_collection} doesn't exist, get returns None")
|
||||
return None
|
||||
else:
|
||||
# For other API errors, log and re-raise
|
||||
_, error_msg = self._extract_error_message(e)
|
||||
log.warning(f"Unexpected Qdrant error during get: {error_msg}")
|
||||
raise
|
||||
except Exception as e:
|
||||
# For non-Qdrant exceptions, log and return None
|
||||
log.exception(f"Error getting collection '{collection_name}': {e}")
|
||||
if not self.client.collection_exists(collection_name=mt_collection):
|
||||
log.debug(f"Collection {mt_collection} doesn't exist, get returns None")
|
||||
return None
|
||||
|
||||
def _handle_operation_with_error_retry(
|
||||
self, operation_name, mt_collection, points, dimension
|
||||
):
|
||||
"""
|
||||
Private helper to handle common error cases for insert and upsert operations.
|
||||
|
||||
Args:
|
||||
operation_name: 'insert' or 'upsert'
|
||||
mt_collection: The multi-tenant collection name
|
||||
points: The vector points to insert/upsert
|
||||
dimension: The dimension of the vectors
|
||||
|
||||
Returns:
|
||||
The operation result (for upsert) or None (for insert)
|
||||
"""
|
||||
try:
|
||||
if operation_name == "insert":
|
||||
self.client.upload_points(mt_collection, points)
|
||||
return None
|
||||
else: # upsert
|
||||
return self.client.upsert(mt_collection, points)
|
||||
except (UnexpectedResponse, grpc.RpcError) as e:
|
||||
# Handle collection not found
|
||||
if self._is_collection_not_found_error(e):
|
||||
log.info(
|
||||
f"Collection {mt_collection} doesn't exist. Creating it with dimension {dimension}."
|
||||
)
|
||||
# Create collection with correct dimensions from our vectors
|
||||
self._create_multi_tenant_collection_if_not_exists(
|
||||
mt_collection_name=mt_collection, dimension=dimension
|
||||
)
|
||||
# Try operation again - no need for dimension adjustment since we just created with correct dimensions
|
||||
if operation_name == "insert":
|
||||
self.client.upload_points(mt_collection, points)
|
||||
return None
|
||||
else: # upsert
|
||||
return self.client.upsert(mt_collection, points)
|
||||
|
||||
# Handle dimension mismatch
|
||||
elif self._is_dimension_mismatch_error(e):
|
||||
# For dimension errors, the collection must exist, so get its configuration
|
||||
mt_collection_info = self.client.get_collection(mt_collection)
|
||||
existing_size = mt_collection_info.config.params.vectors.size
|
||||
|
||||
log.info(
|
||||
f"Dimension mismatch: Collection {mt_collection} expects {existing_size}, got {dimension}"
|
||||
)
|
||||
|
||||
if existing_size < dimension:
|
||||
# Truncate vectors to fit
|
||||
log.info(
|
||||
f"Truncating vectors from {dimension} to {existing_size} dimensions"
|
||||
)
|
||||
points = [
|
||||
PointStruct(
|
||||
id=point.id,
|
||||
vector=point.vector[:existing_size],
|
||||
payload=point.payload,
|
||||
)
|
||||
for point in points
|
||||
]
|
||||
elif existing_size > dimension:
|
||||
# Pad vectors with zeros
|
||||
log.info(
|
||||
f"Padding vectors from {dimension} to {existing_size} dimensions with zeros"
|
||||
)
|
||||
points = [
|
||||
PointStruct(
|
||||
id=point.id,
|
||||
vector=point.vector
|
||||
+ [0] * (existing_size - len(point.vector)),
|
||||
payload=point.payload,
|
||||
)
|
||||
for point in points
|
||||
]
|
||||
# Try operation again with adjusted dimensions
|
||||
if operation_name == "insert":
|
||||
self.client.upload_points(mt_collection, points)
|
||||
return None
|
||||
else: # upsert
|
||||
return self.client.upsert(mt_collection, points)
|
||||
else:
|
||||
# Not a known error we can handle, log and re-raise
|
||||
_, error_msg = self._extract_error_message(e)
|
||||
log.warning(f"Unhandled Qdrant error: {error_msg}")
|
||||
raise
|
||||
except Exception as e:
|
||||
# For non-Qdrant exceptions, re-raise
|
||||
raise
|
||||
|
||||
def insert(self, collection_name: str, items: list[VectorItem]):
|
||||
"""
|
||||
Insert items with tenant ID.
|
||||
"""
|
||||
if not self.client or not items:
|
||||
return None
|
||||
|
||||
# Map to multi-tenant collection and tenant ID
|
||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||
|
||||
# Get dimensions from the actual vectors
|
||||
dimension = len(items[0]["vector"]) if items else None
|
||||
|
||||
# Create points with tenant ID
|
||||
points = self._create_points(items, tenant_id)
|
||||
|
||||
# Handle the operation with error retry
|
||||
return self._handle_operation_with_error_retry(
|
||||
"insert", mt_collection, points, dimension
|
||||
tenant_filter = _tenant_filter(tenant_id)
|
||||
points = self.client.query_points(
|
||||
collection_name=mt_collection,
|
||||
query_filter=models.Filter(must=[tenant_filter]),
|
||||
limit=NO_LIMIT,
|
||||
)
|
||||
return self._result_to_get_result(points.points)
|
||||
|
||||
def upsert(self, collection_name: str, items: list[VectorItem]):
|
||||
def upsert(self, collection_name: str, items: List[VectorItem]):
|
||||
"""
|
||||
Upsert items with tenant ID.
|
||||
"""
|
||||
if not self.client or not items:
|
||||
return None
|
||||
|
||||
# Map to multi-tenant collection and tenant ID
|
||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||
|
||||
# Get dimensions from the actual vectors
|
||||
dimension = len(items[0]["vector"]) if items else None
|
||||
|
||||
# Create points with tenant ID
|
||||
dimension = len(items[0]["vector"])
|
||||
self._ensure_collection(mt_collection, dimension)
|
||||
points = self._create_points(items, tenant_id)
|
||||
self.client.upload_points(mt_collection, points)
|
||||
return None
|
||||
|
||||
# Handle the operation with error retry
|
||||
return self._handle_operation_with_error_retry(
|
||||
"upsert", mt_collection, points, dimension
|
||||
)
|
||||
def insert(self, collection_name: str, items: List[VectorItem]):
|
||||
"""
|
||||
Insert items with tenant ID.
|
||||
"""
|
||||
return self.upsert(collection_name, items)
|
||||
|
||||
def reset(self):
|
||||
"""
|
||||
@@ -677,11 +328,9 @@ class QdrantClient(VectorDBBase):
|
||||
"""
|
||||
if not self.client:
|
||||
return None
|
||||
|
||||
collection_names = self.client.get_collections().collections
|
||||
for collection_name in collection_names:
|
||||
if collection_name.name.startswith(self.collection_prefix):
|
||||
self.client.delete_collection(collection_name=collection_name.name)
|
||||
for collection in self.client.get_collections().collections:
|
||||
if collection.name.startswith(self.collection_prefix):
|
||||
self.client.delete_collection(collection_name=collection.name)
|
||||
|
||||
def delete_collection(self, collection_name: str):
|
||||
"""
|
||||
@@ -689,24 +338,13 @@ class QdrantClient(VectorDBBase):
|
||||
"""
|
||||
if not self.client:
|
||||
return None
|
||||
|
||||
# Map to multi-tenant collection and tenant ID
|
||||
mt_collection, tenant_id = self._get_collection_and_tenant_id(collection_name)
|
||||
|
||||
tenant_filter = models.FieldCondition(
|
||||
key="tenant_id", match=models.MatchValue(value=tenant_id)
|
||||
)
|
||||
|
||||
field_conditions = [tenant_filter]
|
||||
|
||||
update_result = self.client.delete(
|
||||
if not self.client.collection_exists(collection_name=mt_collection):
|
||||
log.debug(f"Collection {mt_collection} doesn't exist, nothing to delete")
|
||||
return None
|
||||
self.client.delete(
|
||||
collection_name=mt_collection,
|
||||
points_selector=models.FilterSelector(
|
||||
filter=models.Filter(must=field_conditions)
|
||||
filter=models.Filter(must=[_tenant_filter(tenant_id)])
|
||||
),
|
||||
)
|
||||
|
||||
if self.client.get_collection(mt_collection).points_count == 0:
|
||||
self.client.delete_collection(mt_collection)
|
||||
|
||||
return update_result
|
||||
|
||||
@@ -0,0 +1,745 @@
|
||||
from open_webui.retrieval.vector.main import (
|
||||
VectorDBBase,
|
||||
VectorItem,
|
||||
GetResult,
|
||||
SearchResult,
|
||||
)
|
||||
from open_webui.config import S3_VECTOR_BUCKET_NAME, S3_VECTOR_REGION
|
||||
from open_webui.env import SRC_LOG_LEVELS
|
||||
from typing import List, Optional, Dict, Any, Union
|
||||
import logging
|
||||
import boto3
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
log.setLevel(SRC_LOG_LEVELS["RAG"])
|
||||
|
||||
|
||||
class S3VectorClient(VectorDBBase):
|
||||
"""
|
||||
AWS S3 Vector integration for Open WebUI Knowledge.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.bucket_name = S3_VECTOR_BUCKET_NAME
|
||||
self.region = S3_VECTOR_REGION
|
||||
|
||||
# Simple validation - log warnings instead of raising exceptions
|
||||
if not self.bucket_name:
|
||||
log.warning("S3_VECTOR_BUCKET_NAME not set - S3Vector will not work")
|
||||
if not self.region:
|
||||
log.warning("S3_VECTOR_REGION not set - S3Vector will not work")
|
||||
|
||||
if self.bucket_name and self.region:
|
||||
try:
|
||||
self.client = boto3.client("s3vectors", region_name=self.region)
|
||||
log.info(
|
||||
f"S3Vector client initialized for bucket '{self.bucket_name}' in region '{self.region}'"
|
||||
)
|
||||
except Exception as e:
|
||||
log.error(f"Failed to initialize S3Vector client: {e}")
|
||||
self.client = None
|
||||
else:
|
||||
self.client = None
|
||||
|
||||
def _create_index(
|
||||
self,
|
||||
index_name: str,
|
||||
dimension: int,
|
||||
data_type: str = "float32",
|
||||
distance_metric: str = "cosine",
|
||||
) -> None:
|
||||
"""
|
||||
Create a new index in the S3 vector bucket for the given collection if it does not exist.
|
||||
"""
|
||||
if self.has_collection(index_name):
|
||||
log.debug(f"Index '{index_name}' already exists, skipping creation")
|
||||
return
|
||||
|
||||
try:
|
||||
self.client.create_index(
|
||||
vectorBucketName=self.bucket_name,
|
||||
indexName=index_name,
|
||||
dataType=data_type,
|
||||
dimension=dimension,
|
||||
distanceMetric=distance_metric,
|
||||
)
|
||||
log.info(
|
||||
f"Created S3 index: {index_name} (dim={dimension}, type={data_type}, metric={distance_metric})"
|
||||
)
|
||||
except Exception as e:
|
||||
log.error(f"Error creating S3 index '{index_name}': {e}")
|
||||
raise
|
||||
|
||||
def _filter_metadata(
|
||||
self, metadata: Dict[str, Any], item_id: str
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Filter vector metadata keys to comply with S3 Vector API limit of 10 keys maximum.
|
||||
"""
|
||||
if not isinstance(metadata, dict) or len(metadata) <= 10:
|
||||
return metadata
|
||||
|
||||
# Keep only the first 10 keys, prioritizing important ones based on actual Open WebUI metadata
|
||||
important_keys = [
|
||||
"text", # The actual document content
|
||||
"file_id", # File ID
|
||||
"source", # Document source file
|
||||
"title", # Document title
|
||||
"page", # Page number
|
||||
"total_pages", # Total pages in document
|
||||
"embedding_config", # Embedding configuration
|
||||
"created_by", # User who created it
|
||||
"name", # Document name
|
||||
"hash", # Content hash
|
||||
]
|
||||
filtered_metadata = {}
|
||||
|
||||
# First, add important keys if they exist
|
||||
for key in important_keys:
|
||||
if key in metadata:
|
||||
filtered_metadata[key] = metadata[key]
|
||||
if len(filtered_metadata) >= 10:
|
||||
break
|
||||
|
||||
# If we still have room, add other keys
|
||||
if len(filtered_metadata) < 10:
|
||||
for key, value in metadata.items():
|
||||
if key not in filtered_metadata:
|
||||
filtered_metadata[key] = value
|
||||
if len(filtered_metadata) >= 10:
|
||||
break
|
||||
|
||||
log.warning(
|
||||
f"Metadata for key '{item_id}' had {len(metadata)} keys, limited to 10 keys"
|
||||
)
|
||||
return filtered_metadata
|
||||
|
||||
def has_collection(self, collection_name: str) -> bool:
|
||||
"""
|
||||
Check if a vector index (collection) exists in the S3 vector bucket.
|
||||
"""
|
||||
|
||||
try:
|
||||
response = self.client.list_indexes(vectorBucketName=self.bucket_name)
|
||||
indexes = response.get("indexes", [])
|
||||
return any(idx.get("indexName") == collection_name for idx in indexes)
|
||||
except Exception as e:
|
||||
log.error(f"Error listing indexes: {e}")
|
||||
return False
|
||||
|
||||
def delete_collection(self, collection_name: str) -> None:
|
||||
"""
|
||||
Delete an entire S3 Vector index/collection.
|
||||
"""
|
||||
|
||||
if not self.has_collection(collection_name):
|
||||
log.warning(
|
||||
f"Collection '{collection_name}' does not exist, nothing to delete"
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
log.info(f"Deleting collection '{collection_name}'")
|
||||
self.client.delete_index(
|
||||
vectorBucketName=self.bucket_name, indexName=collection_name
|
||||
)
|
||||
log.info(f"Successfully deleted collection '{collection_name}'")
|
||||
except Exception as e:
|
||||
log.error(f"Error deleting collection '{collection_name}': {e}")
|
||||
raise
|
||||
|
||||
def insert(self, collection_name: str, items: List[VectorItem]) -> None:
|
||||
"""
|
||||
Insert vector items into the S3 Vector index. Create index if it does not exist.
|
||||
"""
|
||||
if not items:
|
||||
log.warning("No items to insert")
|
||||
return
|
||||
|
||||
dimension = len(items[0]["vector"])
|
||||
|
||||
try:
|
||||
if not self.has_collection(collection_name):
|
||||
log.info(f"Index '{collection_name}' does not exist. Creating index.")
|
||||
self._create_index(
|
||||
index_name=collection_name,
|
||||
dimension=dimension,
|
||||
data_type="float32",
|
||||
distance_metric="cosine",
|
||||
)
|
||||
|
||||
# Prepare vectors for insertion
|
||||
vectors = []
|
||||
for item in items:
|
||||
# Ensure vector data is in the correct format for S3 Vector API
|
||||
vector_data = item["vector"]
|
||||
if isinstance(vector_data, list):
|
||||
# Convert list to float32 values as required by S3 Vector API
|
||||
vector_data = [float(x) for x in vector_data]
|
||||
|
||||
# Prepare metadata, ensuring the text field is preserved
|
||||
metadata = item.get("metadata", {}).copy()
|
||||
|
||||
# Add the text field to metadata so it's available for retrieval
|
||||
metadata["text"] = item["text"]
|
||||
|
||||
# Filter metadata to comply with S3 Vector API limit of 10 keys
|
||||
metadata = self._filter_metadata(metadata, item["id"])
|
||||
|
||||
vectors.append(
|
||||
{
|
||||
"key": item["id"],
|
||||
"data": {"float32": vector_data},
|
||||
"metadata": metadata,
|
||||
}
|
||||
)
|
||||
# Insert vectors
|
||||
self.client.put_vectors(
|
||||
vectorBucketName=self.bucket_name,
|
||||
indexName=collection_name,
|
||||
vectors=vectors,
|
||||
)
|
||||
log.info(f"Inserted {len(vectors)} vectors into index '{collection_name}'.")
|
||||
except Exception as e:
|
||||
log.error(f"Error inserting vectors: {e}")
|
||||
raise
|
||||
|
||||
def upsert(self, collection_name: str, items: List[VectorItem]) -> None:
|
||||
"""
|
||||
Insert or update vector items in the S3 Vector index. Create index if it does not exist.
|
||||
"""
|
||||
if not items:
|
||||
log.warning("No items to upsert")
|
||||
return
|
||||
|
||||
dimension = len(items[0]["vector"])
|
||||
log.info(f"Upsert dimension: {dimension}")
|
||||
|
||||
try:
|
||||
if not self.has_collection(collection_name):
|
||||
log.info(
|
||||
f"Index '{collection_name}' does not exist. Creating index for upsert."
|
||||
)
|
||||
self._create_index(
|
||||
index_name=collection_name,
|
||||
dimension=dimension,
|
||||
data_type="float32",
|
||||
distance_metric="cosine",
|
||||
)
|
||||
|
||||
# Prepare vectors for upsert
|
||||
vectors = []
|
||||
for item in items:
|
||||
# Ensure vector data is in the correct format for S3 Vector API
|
||||
vector_data = item["vector"]
|
||||
if isinstance(vector_data, list):
|
||||
# Convert list to float32 values as required by S3 Vector API
|
||||
vector_data = [float(x) for x in vector_data]
|
||||
|
||||
# Prepare metadata, ensuring the text field is preserved
|
||||
metadata = item.get("metadata", {}).copy()
|
||||
# Add the text field to metadata so it's available for retrieval
|
||||
metadata["text"] = item["text"]
|
||||
|
||||
# Filter metadata to comply with S3 Vector API limit of 10 keys
|
||||
metadata = self._filter_metadata(metadata, item["id"])
|
||||
|
||||
vectors.append(
|
||||
{
|
||||
"key": item["id"],
|
||||
"data": {"float32": vector_data},
|
||||
"metadata": metadata,
|
||||
}
|
||||
)
|
||||
# Upsert vectors (using put_vectors for upsert semantics)
|
||||
log.info(
|
||||
f"Upserting {len(vectors)} vectors. First vector sample: key={vectors[0]['key']}, data_type={type(vectors[0]['data']['float32'])}, data_len={len(vectors[0]['data']['float32'])}"
|
||||
)
|
||||
self.client.put_vectors(
|
||||
vectorBucketName=self.bucket_name,
|
||||
indexName=collection_name,
|
||||
vectors=vectors,
|
||||
)
|
||||
log.info(f"Upserted {len(vectors)} vectors into index '{collection_name}'.")
|
||||
except Exception as e:
|
||||
log.error(f"Error upserting vectors: {e}")
|
||||
raise
|
||||
|
||||
def search(
|
||||
self, collection_name: str, vectors: List[List[Union[float, int]]], limit: int
|
||||
) -> Optional[SearchResult]:
|
||||
"""
|
||||
Search for similar vectors in a collection using multiple query vectors.
|
||||
"""
|
||||
|
||||
if not self.has_collection(collection_name):
|
||||
log.warning(f"Collection '{collection_name}' does not exist")
|
||||
return None
|
||||
|
||||
if not vectors:
|
||||
log.warning("No query vectors provided")
|
||||
return None
|
||||
|
||||
try:
|
||||
log.info(
|
||||
f"Searching collection '{collection_name}' with {len(vectors)} query vectors, limit={limit}"
|
||||
)
|
||||
|
||||
# Initialize result lists
|
||||
all_ids = []
|
||||
all_documents = []
|
||||
all_metadatas = []
|
||||
all_distances = []
|
||||
|
||||
# Process each query vector
|
||||
for i, query_vector in enumerate(vectors):
|
||||
log.debug(f"Processing query vector {i+1}/{len(vectors)}")
|
||||
|
||||
# Prepare the query vector in S3 Vector format
|
||||
query_vector_dict = {"float32": [float(x) for x in query_vector]}
|
||||
|
||||
# Call S3 Vector query API
|
||||
response = self.client.query_vectors(
|
||||
vectorBucketName=self.bucket_name,
|
||||
indexName=collection_name,
|
||||
topK=limit,
|
||||
queryVector=query_vector_dict,
|
||||
returnMetadata=True,
|
||||
returnDistance=True,
|
||||
)
|
||||
|
||||
# Process results for this query
|
||||
query_ids = []
|
||||
query_documents = []
|
||||
query_metadatas = []
|
||||
query_distances = []
|
||||
|
||||
result_vectors = response.get("vectors", [])
|
||||
|
||||
for vector in result_vectors:
|
||||
vector_id = vector.get("key")
|
||||
vector_metadata = vector.get("metadata", {})
|
||||
vector_distance = vector.get("distance", 0.0)
|
||||
|
||||
# Extract document text from metadata
|
||||
document_text = ""
|
||||
if isinstance(vector_metadata, dict):
|
||||
# Get the text field first (highest priority)
|
||||
document_text = vector_metadata.get("text")
|
||||
if not document_text:
|
||||
# Fallback to other possible text fields
|
||||
document_text = (
|
||||
vector_metadata.get("content")
|
||||
or vector_metadata.get("document")
|
||||
or vector_id
|
||||
)
|
||||
else:
|
||||
document_text = vector_id
|
||||
|
||||
query_ids.append(vector_id)
|
||||
query_documents.append(document_text)
|
||||
query_metadatas.append(vector_metadata)
|
||||
query_distances.append(vector_distance)
|
||||
|
||||
# Add this query's results to the overall results
|
||||
all_ids.append(query_ids)
|
||||
all_documents.append(query_documents)
|
||||
all_metadatas.append(query_metadatas)
|
||||
all_distances.append(query_distances)
|
||||
|
||||
log.info(f"Search completed. Found results for {len(all_ids)} queries")
|
||||
|
||||
# Return SearchResult format
|
||||
return SearchResult(
|
||||
ids=all_ids if all_ids else None,
|
||||
documents=all_documents if all_documents else None,
|
||||
metadatas=all_metadatas if all_metadatas else None,
|
||||
distances=all_distances if all_distances else None,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"Error searching collection '{collection_name}': {str(e)}")
|
||||
# Handle specific AWS exceptions
|
||||
if hasattr(e, "response") and "Error" in e.response:
|
||||
error_code = e.response["Error"]["Code"]
|
||||
if error_code == "NotFoundException":
|
||||
log.warning(f"Collection '{collection_name}' not found")
|
||||
return None
|
||||
elif error_code == "ValidationException":
|
||||
log.error(f"Invalid query vector dimensions or parameters")
|
||||
return None
|
||||
elif error_code == "AccessDeniedException":
|
||||
log.error(
|
||||
f"Access denied for collection '{collection_name}'. Check permissions."
|
||||
)
|
||||
return None
|
||||
raise
|
||||
|
||||
def query(
|
||||
self, collection_name: str, filter: Dict, limit: Optional[int] = None
|
||||
) -> Optional[GetResult]:
|
||||
"""
|
||||
Query vectors from a collection using metadata filter.
|
||||
"""
|
||||
|
||||
if not self.has_collection(collection_name):
|
||||
log.warning(f"Collection '{collection_name}' does not exist")
|
||||
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
|
||||
|
||||
if not filter:
|
||||
log.warning("No filter provided, returning all vectors")
|
||||
return self.get(collection_name)
|
||||
|
||||
try:
|
||||
log.info(f"Querying collection '{collection_name}' with filter: {filter}")
|
||||
|
||||
# For S3 Vector, we need to use list_vectors and then filter results
|
||||
# Since S3 Vector may not support complex server-side filtering,
|
||||
# we'll retrieve all vectors and filter client-side
|
||||
|
||||
# Get all vectors first
|
||||
all_vectors_result = self.get(collection_name)
|
||||
|
||||
if not all_vectors_result or not all_vectors_result.ids:
|
||||
log.warning("No vectors found in collection")
|
||||
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
|
||||
|
||||
# Extract the lists from the result
|
||||
all_ids = all_vectors_result.ids[0] if all_vectors_result.ids else []
|
||||
all_documents = (
|
||||
all_vectors_result.documents[0] if all_vectors_result.documents else []
|
||||
)
|
||||
all_metadatas = (
|
||||
all_vectors_result.metadatas[0] if all_vectors_result.metadatas else []
|
||||
)
|
||||
|
||||
# Apply client-side filtering
|
||||
filtered_ids = []
|
||||
filtered_documents = []
|
||||
filtered_metadatas = []
|
||||
|
||||
for i, metadata in enumerate(all_metadatas):
|
||||
if self._matches_filter(metadata, filter):
|
||||
if i < len(all_ids):
|
||||
filtered_ids.append(all_ids[i])
|
||||
if i < len(all_documents):
|
||||
filtered_documents.append(all_documents[i])
|
||||
filtered_metadatas.append(metadata)
|
||||
|
||||
# Apply limit if specified
|
||||
if limit and len(filtered_ids) >= limit:
|
||||
break
|
||||
|
||||
log.info(
|
||||
f"Filter applied: {len(filtered_ids)} vectors match out of {len(all_ids)} total"
|
||||
)
|
||||
|
||||
# Return GetResult format
|
||||
if filtered_ids:
|
||||
return GetResult(
|
||||
ids=[filtered_ids],
|
||||
documents=[filtered_documents],
|
||||
metadatas=[filtered_metadatas],
|
||||
)
|
||||
else:
|
||||
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"Error querying collection '{collection_name}': {str(e)}")
|
||||
# Handle specific AWS exceptions
|
||||
if hasattr(e, "response") and "Error" in e.response:
|
||||
error_code = e.response["Error"]["Code"]
|
||||
if error_code == "NotFoundException":
|
||||
log.warning(f"Collection '{collection_name}' not found")
|
||||
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
|
||||
elif error_code == "AccessDeniedException":
|
||||
log.error(
|
||||
f"Access denied for collection '{collection_name}'. Check permissions."
|
||||
)
|
||||
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
|
||||
raise
|
||||
|
||||
def get(self, collection_name: str) -> Optional[GetResult]:
|
||||
"""
|
||||
Retrieve all vectors from a collection.
|
||||
"""
|
||||
|
||||
if not self.has_collection(collection_name):
|
||||
log.warning(f"Collection '{collection_name}' does not exist")
|
||||
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
|
||||
|
||||
try:
|
||||
log.info(f"Retrieving all vectors from collection '{collection_name}'")
|
||||
|
||||
# Initialize result lists
|
||||
all_ids = []
|
||||
all_documents = []
|
||||
all_metadatas = []
|
||||
|
||||
# Handle pagination
|
||||
next_token = None
|
||||
|
||||
while True:
|
||||
# Prepare request parameters
|
||||
request_params = {
|
||||
"vectorBucketName": self.bucket_name,
|
||||
"indexName": collection_name,
|
||||
"returnData": False, # Don't include vector data (not needed for get)
|
||||
"returnMetadata": True, # Include metadata
|
||||
"maxResults": 500, # Use reasonable page size
|
||||
}
|
||||
|
||||
if next_token:
|
||||
request_params["nextToken"] = next_token
|
||||
|
||||
# Call S3 Vector API
|
||||
response = self.client.list_vectors(**request_params)
|
||||
|
||||
# Process vectors in this page
|
||||
vectors = response.get("vectors", [])
|
||||
|
||||
for vector in vectors:
|
||||
vector_id = vector.get("key")
|
||||
vector_data = vector.get("data", {})
|
||||
vector_metadata = vector.get("metadata", {})
|
||||
|
||||
# Extract the actual vector array
|
||||
vector_array = vector_data.get("float32", [])
|
||||
|
||||
# For documents, we try to extract text from metadata or use the vector ID
|
||||
document_text = ""
|
||||
if isinstance(vector_metadata, dict):
|
||||
# Get the text field first (highest priority)
|
||||
document_text = vector_metadata.get("text")
|
||||
if not document_text:
|
||||
# Fallback to other possible text fields
|
||||
document_text = (
|
||||
vector_metadata.get("content")
|
||||
or vector_metadata.get("document")
|
||||
or vector_id
|
||||
)
|
||||
|
||||
# Log the actual content for debugging
|
||||
log.debug(
|
||||
f"Document text preview (first 200 chars): {str(document_text)[:200]}"
|
||||
)
|
||||
else:
|
||||
document_text = vector_id
|
||||
|
||||
all_ids.append(vector_id)
|
||||
all_documents.append(document_text)
|
||||
all_metadatas.append(vector_metadata)
|
||||
|
||||
# Check if there are more pages
|
||||
next_token = response.get("nextToken")
|
||||
if not next_token:
|
||||
break
|
||||
|
||||
log.info(
|
||||
f"Retrieved {len(all_ids)} vectors from collection '{collection_name}'"
|
||||
)
|
||||
|
||||
# Return in GetResult format
|
||||
# The Open WebUI GetResult expects lists of lists, so we wrap each list
|
||||
if all_ids:
|
||||
return GetResult(
|
||||
ids=[all_ids], documents=[all_documents], metadatas=[all_metadatas]
|
||||
)
|
||||
else:
|
||||
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
|
||||
|
||||
except Exception as e:
|
||||
log.error(
|
||||
f"Error retrieving vectors from collection '{collection_name}': {str(e)}"
|
||||
)
|
||||
# Handle specific AWS exceptions
|
||||
if hasattr(e, "response") and "Error" in e.response:
|
||||
error_code = e.response["Error"]["Code"]
|
||||
if error_code == "NotFoundException":
|
||||
log.warning(f"Collection '{collection_name}' not found")
|
||||
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
|
||||
elif error_code == "AccessDeniedException":
|
||||
log.error(
|
||||
f"Access denied for collection '{collection_name}'. Check permissions."
|
||||
)
|
||||
return GetResult(ids=[[]], documents=[[]], metadatas=[[]])
|
||||
raise
|
||||
|
||||
def delete(
|
||||
self,
|
||||
collection_name: str,
|
||||
ids: Optional[List[str]] = None,
|
||||
filter: Optional[Dict] = None,
|
||||
) -> None:
|
||||
"""
|
||||
Delete vectors by ID or filter from a collection.
|
||||
"""
|
||||
|
||||
if not self.has_collection(collection_name):
|
||||
log.warning(
|
||||
f"Collection '{collection_name}' does not exist, nothing to delete"
|
||||
)
|
||||
return
|
||||
|
||||
# Check if this is a knowledge collection (not file-specific)
|
||||
is_knowledge_collection = not collection_name.startswith("file-")
|
||||
|
||||
try:
|
||||
if ids:
|
||||
# Delete by specific vector IDs/keys
|
||||
log.info(
|
||||
f"Deleting {len(ids)} vectors by IDs from collection '{collection_name}'"
|
||||
)
|
||||
self.client.delete_vectors(
|
||||
vectorBucketName=self.bucket_name,
|
||||
indexName=collection_name,
|
||||
keys=ids,
|
||||
)
|
||||
log.info(f"Deleted {len(ids)} vectors from index '{collection_name}'")
|
||||
|
||||
elif filter:
|
||||
# Handle filter-based deletion
|
||||
log.info(
|
||||
f"Deleting vectors by filter from collection '{collection_name}': {filter}"
|
||||
)
|
||||
|
||||
# If this is a knowledge collection and we have a file_id filter,
|
||||
# also clean up the corresponding file-specific collection
|
||||
if is_knowledge_collection and "file_id" in filter:
|
||||
file_id = filter["file_id"]
|
||||
file_collection_name = f"file-{file_id}"
|
||||
if self.has_collection(file_collection_name):
|
||||
log.info(
|
||||
f"Found related file-specific collection '{file_collection_name}', deleting it to prevent duplicates"
|
||||
)
|
||||
self.delete_collection(file_collection_name)
|
||||
|
||||
# For the main collection, implement query-then-delete
|
||||
# First, query to get IDs matching the filter
|
||||
query_result = self.query(collection_name, filter)
|
||||
if query_result and query_result.ids and query_result.ids[0]:
|
||||
matching_ids = query_result.ids[0]
|
||||
log.info(
|
||||
f"Found {len(matching_ids)} vectors matching filter, deleting them"
|
||||
)
|
||||
|
||||
# Delete the matching vectors by ID
|
||||
self.client.delete_vectors(
|
||||
vectorBucketName=self.bucket_name,
|
||||
indexName=collection_name,
|
||||
keys=matching_ids,
|
||||
)
|
||||
log.info(
|
||||
f"Deleted {len(matching_ids)} vectors from index '{collection_name}' using filter"
|
||||
)
|
||||
else:
|
||||
log.warning("No vectors found matching the filter criteria")
|
||||
else:
|
||||
log.warning("No IDs or filter provided for deletion")
|
||||
except Exception as e:
|
||||
log.error(
|
||||
f"Error deleting vectors from collection '{collection_name}': {e}"
|
||||
)
|
||||
raise
|
||||
|
||||
def reset(self) -> None:
|
||||
"""
|
||||
Reset/clear all vector data. For S3 Vector, this deletes all indexes.
|
||||
"""
|
||||
|
||||
try:
|
||||
log.warning(
|
||||
"Reset called - this will delete all vector indexes in the S3 bucket"
|
||||
)
|
||||
|
||||
# List all indexes
|
||||
response = self.client.list_indexes(vectorBucketName=self.bucket_name)
|
||||
indexes = response.get("indexes", [])
|
||||
|
||||
if not indexes:
|
||||
log.warning("No indexes found to delete")
|
||||
return
|
||||
|
||||
# Delete all indexes
|
||||
deleted_count = 0
|
||||
for index in indexes:
|
||||
index_name = index.get("indexName")
|
||||
if index_name:
|
||||
try:
|
||||
self.client.delete_index(
|
||||
vectorBucketName=self.bucket_name, indexName=index_name
|
||||
)
|
||||
deleted_count += 1
|
||||
log.info(f"Deleted index: {index_name}")
|
||||
except Exception as e:
|
||||
log.error(f"Error deleting index '{index_name}': {e}")
|
||||
|
||||
log.info(f"Reset completed: deleted {deleted_count} indexes")
|
||||
|
||||
except Exception as e:
|
||||
log.error(f"Error during reset: {e}")
|
||||
raise
|
||||
|
||||
def _matches_filter(self, metadata: Dict[str, Any], filter: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
Check if metadata matches the given filter conditions.
|
||||
"""
|
||||
if not isinstance(metadata, dict) or not isinstance(filter, dict):
|
||||
return False
|
||||
|
||||
# Check each filter condition
|
||||
for key, expected_value in filter.items():
|
||||
# Handle special operators
|
||||
if key.startswith("$"):
|
||||
if key == "$and":
|
||||
# All conditions must match
|
||||
if not isinstance(expected_value, list):
|
||||
continue
|
||||
for condition in expected_value:
|
||||
if not self._matches_filter(metadata, condition):
|
||||
return False
|
||||
elif key == "$or":
|
||||
# At least one condition must match
|
||||
if not isinstance(expected_value, list):
|
||||
continue
|
||||
any_match = False
|
||||
for condition in expected_value:
|
||||
if self._matches_filter(metadata, condition):
|
||||
any_match = True
|
||||
break
|
||||
if not any_match:
|
||||
return False
|
||||
continue
|
||||
|
||||
# Get the actual value from metadata
|
||||
actual_value = metadata.get(key)
|
||||
|
||||
# Handle different types of expected values
|
||||
if isinstance(expected_value, dict):
|
||||
# Handle comparison operators
|
||||
for op, op_value in expected_value.items():
|
||||
if op == "$eq":
|
||||
if actual_value != op_value:
|
||||
return False
|
||||
elif op == "$ne":
|
||||
if actual_value == op_value:
|
||||
return False
|
||||
elif op == "$in":
|
||||
if (
|
||||
not isinstance(op_value, list)
|
||||
or actual_value not in op_value
|
||||
):
|
||||
return False
|
||||
elif op == "$nin":
|
||||
if isinstance(op_value, list) and actual_value in op_value:
|
||||
return False
|
||||
elif op == "$exists":
|
||||
if bool(op_value) != (key in metadata):
|
||||
return False
|
||||
# Add more operators as needed
|
||||
else:
|
||||
# Simple equality check
|
||||
if actual_value != expected_value:
|
||||
return False
|
||||
|
||||
return True
|
||||
@@ -30,6 +30,10 @@ class Vector:
|
||||
from open_webui.retrieval.vector.dbs.pinecone import PineconeClient
|
||||
|
||||
return PineconeClient()
|
||||
case VectorType.S3VECTOR:
|
||||
from open_webui.retrieval.vector.dbs.s3vector import S3VectorClient
|
||||
|
||||
return S3VectorClient()
|
||||
case VectorType.OPENSEARCH:
|
||||
from open_webui.retrieval.vector.dbs.opensearch import OpenSearchClient
|
||||
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def stringify_metadata(
|
||||
metadata: dict[str, any],
|
||||
) -> dict[str, any]:
|
||||
for key, value in metadata.items():
|
||||
if (
|
||||
isinstance(value, datetime)
|
||||
or isinstance(value, list)
|
||||
or isinstance(value, dict)
|
||||
):
|
||||
metadata[key] = str(value)
|
||||
return metadata
|
||||
Reference in New Issue
Block a user