Emil_Shanaty/open-webui-ai4me
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: format

Browse Source
This commit is contained in:
Timothy Jaeryang Baek
2025-08-21 04:47:28 +04:00
parent efcf6db687
commit 1a15a62b73
69 changed files with 1995 additions and 775 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/open_webui/retrieval/vector/dbs/s3vector.py
+19 -11
View File
@@ -197,19 +197,23 @@ class S3VectorClient(VectorDBBase):
"metadata": metadata,
}
)
# Insert vectors in batches of 500 (S3 Vector API limit)
batch_size = 500
for i in range(0, len(vectors), batch_size):
batch = vectors[i:i + batch_size]
batch = vectors[i : i + batch_size]
self.client.put_vectors(
vectorBucketName=self.bucket_name,
indexName=collection_name,
vectors=batch,
)
log.info(f"Inserted batch {i//batch_size + 1}: {len(batch)} vectors into index '{collection_name}'.")
log.info(f"Completed insertion of {len(vectors)} vectors into index '{collection_name}'.")
log.info(
f"Inserted batch {i//batch_size + 1}: {len(batch)} vectors into index '{collection_name}'."
)
log.info(
f"Completed insertion of {len(vectors)} vectors into index '{collection_name}'."
)
except Exception as e:
log.error(f"Error inserting vectors: {e}")
raise
@@ -264,25 +268,29 @@ class S3VectorClient(VectorDBBase):
"metadata": metadata,
}
)
# Upsert vectors in batches of 500 (S3 Vector API limit)
batch_size = 500
for i in range(0, len(vectors), batch_size):
batch = vectors[i:i + batch_size]
batch = vectors[i : i + batch_size]
if i == 0: # Log sample info for first batch only
log.info(
f"Upserting batch 1: {len(batch)} vectors. First vector sample: key={batch[0]['key']}, data_type={type(batch[0]['data']['float32'])}, data_len={len(batch[0]['data']['float32'])}"
)
else:
log.info(f"Upserting batch {i//batch_size + 1}: {len(batch)} vectors.")
log.info(
f"Upserting batch {i//batch_size + 1}: {len(batch)} vectors."
)
self.client.put_vectors(
vectorBucketName=self.bucket_name,
indexName=collection_name,
vectors=batch,
)
log.info(f"Completed upsert of {len(vectors)} vectors into index '{collection_name}'.")
log.info(
f"Completed upsert of {len(vectors)} vectors into index '{collection_name}'."
)
except Exception as e:
log.error(f"Error upserting vectors: {e}")
raise