Update utils.py (#21105)

This commit is contained in:
Classic298
2026-02-13 13:37:12 -06:00
committed by GitHub
parent d01b1d4880
commit 97a3b1528d
+10 -10
View File
@@ -4,6 +4,7 @@ KEYS_TO_EXCLUDE = ["content", "pages", "tables", "paragraphs", "sections", "figu
def filter_metadata(metadata: dict[str, any]) -> dict[str, any]: def filter_metadata(metadata: dict[str, any]) -> dict[str, any]:
# Removes large/redundant fields from metadata dict.
metadata = { metadata = {
key: value for key, value in metadata.items() if key not in KEYS_TO_EXCLUDE key: value for key, value in metadata.items() if key not in KEYS_TO_EXCLUDE
} }
@@ -13,16 +14,15 @@ def filter_metadata(metadata: dict[str, any]) -> dict[str, any]:
def process_metadata( def process_metadata(
metadata: dict[str, any], metadata: dict[str, any],
) -> dict[str, any]: ) -> dict[str, any]:
# Removes large fields and converts non-serializable types (datetime, list, dict) to strings.
result = {}
for key, value in metadata.items(): for key, value in metadata.items():
# Remove large fields # Skip large fields
if key in KEYS_TO_EXCLUDE: if key in KEYS_TO_EXCLUDE:
del metadata[key] continue
# Convert non-serializable fields to strings # Convert non-serializable fields to strings
if ( if isinstance(value, (datetime, list, dict)):
isinstance(value, datetime) result[key] = str(value)
or isinstance(value, list) else:
or isinstance(value, dict) result[key] = value
): return result
metadata[key] = str(value)
return metadata