refac: async db

This commit is contained in:
Timothy Jaeryang Baek
2026-04-12 14:22:11 -05:00
parent b618d84065
commit 27169124f2
74 changed files with 4831 additions and 4479 deletions
+88 -80
View File
@@ -2,14 +2,15 @@ import logging
import time
from typing import Optional
from sqlalchemy.orm import Session
from open_webui.internal.db import Base, get_db, get_db_context
from open_webui.models.users import Users, UserResponse
from sqlalchemy import select, delete, update, or_
from sqlalchemy.ext.asyncio import AsyncSession
from open_webui.internal.db import Base, get_async_db_context
from open_webui.models.users import Users, User, UserModel, UserResponse
from open_webui.models.groups import Groups
from open_webui.models.access_grants import AccessGrantModel, AccessGrants
from pydantic import BaseModel, ConfigDict, Field
from sqlalchemy import JSON, BigInteger, Boolean, Column, String, Text, or_
from sqlalchemy import JSON, BigInteger, Boolean, Column, String, Text, func
log = logging.getLogger(__name__)
@@ -105,28 +106,28 @@ class SkillAccessListResponse(BaseModel):
class SkillsTable:
def _get_access_grants(self, skill_id: str, db: Optional[Session] = None) -> list[AccessGrantModel]:
return AccessGrants.get_grants_by_resource('skill', skill_id, db=db)
async def _get_access_grants(self, skill_id: str, db: Optional[AsyncSession] = None) -> list[AccessGrantModel]:
return await AccessGrants.get_grants_by_resource('skill', skill_id, db=db)
def _to_skill_model(
async def _to_skill_model(
self,
skill: Skill,
access_grants: Optional[list[AccessGrantModel]] = None,
db: Optional[Session] = None,
db: Optional[AsyncSession] = None,
) -> SkillModel:
skill_data = SkillModel.model_validate(skill).model_dump(exclude={'access_grants'})
skill_data['access_grants'] = (
access_grants if access_grants is not None else self._get_access_grants(skill_data['id'], db=db)
access_grants if access_grants is not None else await self._get_access_grants(skill_data['id'], db=db)
)
return SkillModel.model_validate(skill_data)
def insert_new_skill(
async def insert_new_skill(
self,
user_id: str,
form_data: SkillForm,
db: Optional[Session] = None,
db: Optional[AsyncSession] = None,
) -> Optional[SkillModel]:
with get_db_context(db) as db:
async with get_async_db_context(db) as db:
try:
result = Skill(
**{
@@ -137,43 +138,45 @@ class SkillsTable:
}
)
db.add(result)
db.commit()
db.refresh(result)
AccessGrants.set_access_grants('skill', result.id, form_data.access_grants, db=db)
await db.commit()
await db.refresh(result)
await AccessGrants.set_access_grants('skill', result.id, form_data.access_grants, db=db)
if result:
return self._to_skill_model(result, db=db)
return await self._to_skill_model(result, db=db)
else:
return None
except Exception as e:
log.exception(f'Error creating a new skill: {e}')
return None
def get_skill_by_id(self, id: str, db: Optional[Session] = None) -> Optional[SkillModel]:
async def get_skill_by_id(self, id: str, db: Optional[AsyncSession] = None) -> Optional[SkillModel]:
try:
with get_db_context(db) as db:
skill = db.get(Skill, id)
return self._to_skill_model(skill, db=db) if skill else None
async with get_async_db_context(db) as db:
skill = await db.get(Skill, id)
return await self._to_skill_model(skill, db=db) if skill else None
except Exception:
return None
def get_skill_by_name(self, name: str, db: Optional[Session] = None) -> Optional[SkillModel]:
async def get_skill_by_name(self, name: str, db: Optional[AsyncSession] = None) -> Optional[SkillModel]:
try:
with get_db_context(db) as db:
skill = db.query(Skill).filter_by(name=name).first()
return self._to_skill_model(skill, db=db) if skill else None
async with get_async_db_context(db) as db:
result = await db.execute(select(Skill).filter_by(name=name))
skill = result.scalars().first()
return await self._to_skill_model(skill, db=db) if skill else None
except Exception:
return None
def get_skills(self, db: Optional[Session] = None) -> list[SkillUserModel]:
with get_db_context(db) as db:
all_skills = db.query(Skill).order_by(Skill.updated_at.desc()).all()
async def get_skills(self, db: Optional[AsyncSession] = None) -> list[SkillUserModel]:
async with get_async_db_context(db) as db:
result = await db.execute(select(Skill).order_by(Skill.updated_at.desc()))
all_skills = result.scalars().all()
user_ids = list(set(skill.user_id for skill in all_skills))
skill_ids = [skill.id for skill in all_skills]
users = Users.get_users_by_user_ids(user_ids, db=db) if user_ids else []
users = await Users.get_users_by_user_ids(user_ids, db=db) if user_ids else []
users_dict = {user.id: user for user in users}
grants_map = AccessGrants.get_grants_by_resources('skill', skill_ids, db=db)
grants_map = await AccessGrants.get_grants_by_resources('skill', skill_ids, db=db)
skills = []
for skill in all_skills:
@@ -181,56 +184,56 @@ class SkillsTable:
skills.append(
SkillUserModel.model_validate(
{
**self._to_skill_model(
**(await self._to_skill_model(
skill,
access_grants=grants_map.get(skill.id, []),
db=db,
).model_dump(),
)).model_dump(),
'user': user.model_dump() if user else None,
}
)
)
return skills
def get_skills_by_user_id(
self, user_id: str, permission: str = 'write', db: Optional[Session] = None
async def get_skills_by_user_id(
self, user_id: str, permission: str = 'write', db: Optional[AsyncSession] = None
) -> list[SkillUserModel]:
skills = self.get_skills(db=db)
user_group_ids = {group.id for group in Groups.get_groups_by_member_id(user_id, db=db)}
skills = await self.get_skills(db=db)
user_groups = await Groups.get_groups_by_member_id(user_id, db=db)
user_group_ids = {group.id for group in user_groups}
return [
skill
for skill in skills
if skill.user_id == user_id
or AccessGrants.has_access(
result = []
for skill in skills:
if skill.user_id == user_id:
result.append(skill)
elif await AccessGrants.has_access(
user_id=user_id,
resource_type='skill',
resource_id=skill.id,
permission=permission,
user_group_ids=user_group_ids,
db=db,
)
]
):
result.append(skill)
return result
def search_skills(
async def search_skills(
self,
user_id: str,
filter: dict = {},
skip: int = 0,
limit: int = 30,
db: Optional[Session] = None,
db: Optional[AsyncSession] = None,
) -> SkillListResponse:
try:
with get_db_context(db) as db:
from open_webui.models.users import User, UserModel
async with get_async_db_context(db) as db:
# Join with User table for user filtering
query = db.query(Skill, User).outerjoin(User, User.id == Skill.user_id)
stmt = select(Skill, User).outerjoin(User, User.id == Skill.user_id)
if filter:
query_key = filter.get('query')
if query_key:
query = query.filter(
stmt = stmt.filter(
or_(
Skill.name.ilike(f'%{query_key}%'),
Skill.description.ilike(f'%{query_key}%'),
@@ -242,44 +245,48 @@ class SkillsTable:
view_option = filter.get('view_option')
if view_option == 'created':
query = query.filter(Skill.user_id == user_id)
stmt = stmt.filter(Skill.user_id == user_id)
elif view_option == 'shared':
query = query.filter(Skill.user_id != user_id)
stmt = stmt.filter(Skill.user_id != user_id)
# Apply access grant filtering
query = AccessGrants.has_permission_filter(
stmt = AccessGrants.has_permission_filter(
db=db,
query=query,
query=stmt,
DocumentModel=Skill,
filter=filter,
resource_type='skill',
permission='read',
)
query = query.order_by(Skill.updated_at.desc())
stmt = stmt.order_by(Skill.updated_at.desc())
# Count BEFORE pagination
total = query.count()
count_result = await db.execute(
select(func.count()).select_from(stmt.subquery())
)
total = count_result.scalar()
if skip:
query = query.offset(skip)
stmt = stmt.offset(skip)
if limit:
query = query.limit(limit)
stmt = stmt.limit(limit)
items = query.all()
result = await db.execute(stmt)
items = result.all()
skill_ids = [skill.id for skill, _ in items]
grants_map = AccessGrants.get_grants_by_resources('skill', skill_ids, db=db)
grants_map = await AccessGrants.get_grants_by_resources('skill', skill_ids, db=db)
skills = []
for skill, user in items:
skills.append(
SkillUserResponse(
**self._to_skill_model(
**(await self._to_skill_model(
skill,
access_grants=grants_map.get(skill.id, []),
db=db,
).model_dump(),
)).model_dump(),
user=(UserResponse(**UserModel.model_validate(user).model_dump()) if user else None),
)
)
@@ -289,43 +296,44 @@ class SkillsTable:
log.exception(f'Error searching skills: {e}')
return SkillListResponse(items=[], total=0)
def update_skill_by_id(self, id: str, updated: dict, db: Optional[Session] = None) -> Optional[SkillModel]:
async def update_skill_by_id(self, id: str, updated: dict, db: Optional[AsyncSession] = None) -> Optional[SkillModel]:
try:
with get_db_context(db) as db:
async with get_async_db_context(db) as db:
access_grants = updated.pop('access_grants', None)
db.query(Skill).filter_by(id=id).update({**updated, 'updated_at': int(time.time())})
db.commit()
await db.execute(update(Skill).filter_by(id=id).values(**updated, updated_at=int(time.time())))
await db.commit()
if access_grants is not None:
AccessGrants.set_access_grants('skill', id, access_grants, db=db)
await AccessGrants.set_access_grants('skill', id, access_grants, db=db)
skill = db.query(Skill).get(id)
db.refresh(skill)
return self._to_skill_model(skill, db=db)
skill = await db.get(Skill, id)
await db.refresh(skill)
return await self._to_skill_model(skill, db=db)
except Exception:
return None
def toggle_skill_by_id(self, id: str, db: Optional[Session] = None) -> Optional[SkillModel]:
with get_db_context(db) as db:
async def toggle_skill_by_id(self, id: str, db: Optional[AsyncSession] = None) -> Optional[SkillModel]:
async with get_async_db_context(db) as db:
try:
skill = db.query(Skill).filter_by(id=id).first()
result = await db.execute(select(Skill).filter_by(id=id))
skill = result.scalars().first()
if not skill:
return None
skill.is_active = not skill.is_active
skill.updated_at = int(time.time())
db.commit()
db.refresh(skill)
await db.commit()
await db.refresh(skill)
return self._to_skill_model(skill, db=db)
return await self._to_skill_model(skill, db=db)
except Exception:
return None
def delete_skill_by_id(self, id: str, db: Optional[Session] = None) -> bool:
async def delete_skill_by_id(self, id: str, db: Optional[AsyncSession] = None) -> bool:
try:
with get_db_context(db) as db:
AccessGrants.revoke_all_access('skill', id, db=db)
db.query(Skill).filter_by(id=id).delete()
db.commit()
async with get_async_db_context(db) as db:
await AccessGrants.revoke_all_access('skill', id, db=db)
await db.execute(delete(Skill).filter_by(id=id))
await db.commit()
return True
except Exception: