refac

backend/open_webui/socket/utils.py

@@ -118,6 +118,8 @@ class RedisDict:
 
 
 class YdocManager:
+    COMPACTION_THRESHOLD = 500
+
     def __init__(
         self,
         redis=None,
@@ -133,10 +135,42 @@ class YdocManager:
         if self._redis:
             redis_key = f"{self._redis_key_prefix}:{document_id}:updates"
             await self._redis.rpush(redis_key, json.dumps(list(update)))
+            list_len = await self._redis.llen(redis_key)
+            if list_len >= self.COMPACTION_THRESHOLD:
+                await self._compact_updates_redis(document_id)
         else:
             if document_id not in self._updates:
                 self._updates[document_id] = []
             self._updates[document_id].append(update)
+            if len(self._updates[document_id]) >= self.COMPACTION_THRESHOLD:
+                self._compact_updates_memory(document_id)
+
+    async def _compact_updates_redis(self, document_id: str):
+        """Rolling compaction: squash oldest half into one snapshot."""
+        redis_key = f"{self._redis_key_prefix}:{document_id}:updates"
+        all_updates = await self._redis.lrange(redis_key, 0, -1)
+        if len(all_updates) <= 1:
+            return
+        mid = len(all_updates) // 2
+        ydoc = Y.Doc()
+        for raw in all_updates[:mid]:
+            ydoc.apply_update(bytes(json.loads(raw)))
+        snapshot = json.dumps(list(ydoc.get_update()))
+        pipe = self._redis.pipeline()
+        pipe.delete(redis_key)
+        pipe.rpush(redis_key, snapshot, *all_updates[mid:])
+        await pipe.execute()
+
+    def _compact_updates_memory(self, document_id: str):
+        """Rolling compaction: squash oldest half into one snapshot."""
+        updates = self._updates.get(document_id, [])
+        if len(updates) <= 1:
+            return
+        mid = len(updates) // 2
+        ydoc = Y.Doc()
+        for update in updates[:mid]:
+            ydoc.apply_update(bytes(update))
+        self._updates[document_id] = [ydoc.get_update()] + updates[mid:]
 
     async def get_updates(self, document_id: str) -> List[bytes]:
         document_id = document_id.replace(":", "_")