This commit is contained in:
Timothy Jaeryang Baek
2026-03-17 17:58:01 -05:00
parent fcf7208352
commit de3317e26b
220 changed files with 17200 additions and 22836 deletions
+53 -94
View File
@@ -19,7 +19,7 @@ log = logging.getLogger(__name__)
class Feedback(Base):
__tablename__ = "feedback"
__tablename__ = 'feedback'
id = Column(Text, primary_key=True, unique=True)
user_id = Column(Text)
version = Column(BigInteger, default=0)
@@ -81,7 +81,7 @@ class RatingData(BaseModel):
sibling_model_ids: Optional[list[str]] = None
reason: Optional[str] = None
comment: Optional[str] = None
model_config = ConfigDict(extra="allow", protected_namespaces=())
model_config = ConfigDict(extra='allow', protected_namespaces=())
class MetaData(BaseModel):
@@ -89,12 +89,12 @@ class MetaData(BaseModel):
chat_id: Optional[str] = None
message_id: Optional[str] = None
tags: Optional[list[str]] = None
model_config = ConfigDict(extra="allow")
model_config = ConfigDict(extra='allow')
class SnapshotData(BaseModel):
chat: Optional[dict] = None
model_config = ConfigDict(extra="allow")
model_config = ConfigDict(extra='allow')
class FeedbackForm(BaseModel):
@@ -102,14 +102,14 @@ class FeedbackForm(BaseModel):
data: Optional[RatingData] = None
meta: Optional[dict] = None
snapshot: Optional[SnapshotData] = None
model_config = ConfigDict(extra="allow")
model_config = ConfigDict(extra='allow')
class UserResponse(BaseModel):
id: str
name: str
email: str
role: str = "pending"
role: str = 'pending'
last_active_at: int # timestamp in epoch
updated_at: int # timestamp in epoch
@@ -146,12 +146,12 @@ class FeedbackTable:
id = str(uuid.uuid4())
feedback = FeedbackModel(
**{
"id": id,
"user_id": user_id,
"version": 0,
'id': id,
'user_id': user_id,
'version': 0,
**form_data.model_dump(),
"created_at": int(time.time()),
"updated_at": int(time.time()),
'created_at': int(time.time()),
'updated_at': int(time.time()),
}
)
try:
@@ -164,12 +164,10 @@ class FeedbackTable:
else:
return None
except Exception as e:
log.exception(f"Error creating a new feedback: {e}")
log.exception(f'Error creating a new feedback: {e}')
return None
def get_feedback_by_id(
self, id: str, db: Optional[Session] = None
) -> Optional[FeedbackModel]:
def get_feedback_by_id(self, id: str, db: Optional[Session] = None) -> Optional[FeedbackModel]:
try:
with get_db_context(db) as db:
feedback = db.query(Feedback).filter_by(id=id).first()
@@ -191,16 +189,14 @@ class FeedbackTable:
except Exception:
return None
def get_feedbacks_by_chat_id(
self, chat_id: str, db: Optional[Session] = None
) -> list[FeedbackModel]:
def get_feedbacks_by_chat_id(self, chat_id: str, db: Optional[Session] = None) -> list[FeedbackModel]:
"""Get all feedbacks for a specific chat."""
try:
with get_db_context(db) as db:
# meta.chat_id stores the chat reference
feedbacks = (
db.query(Feedback)
.filter(Feedback.meta["chat_id"].as_string() == chat_id)
.filter(Feedback.meta['chat_id'].as_string() == chat_id)
.order_by(Feedback.created_at.desc())
.all()
)
@@ -219,36 +215,28 @@ class FeedbackTable:
query = db.query(Feedback, User).join(User, Feedback.user_id == User.id)
if filter:
order_by = filter.get("order_by")
direction = filter.get("direction")
order_by = filter.get('order_by')
direction = filter.get('direction')
if order_by == "username":
if direction == "asc":
if order_by == 'username':
if direction == 'asc':
query = query.order_by(User.name.asc())
else:
query = query.order_by(User.name.desc())
elif order_by == "model_id":
elif order_by == 'model_id':
# it's stored in feedback.data['model_id']
if direction == "asc":
query = query.order_by(
Feedback.data["model_id"].as_string().asc()
)
if direction == 'asc':
query = query.order_by(Feedback.data['model_id'].as_string().asc())
else:
query = query.order_by(
Feedback.data["model_id"].as_string().desc()
)
elif order_by == "rating":
query = query.order_by(Feedback.data['model_id'].as_string().desc())
elif order_by == 'rating':
# it's stored in feedback.data['rating']
if direction == "asc":
query = query.order_by(
Feedback.data["rating"].as_string().asc()
)
if direction == 'asc':
query = query.order_by(Feedback.data['rating'].as_string().asc())
else:
query = query.order_by(
Feedback.data["rating"].as_string().desc()
)
elif order_by == "updated_at":
if direction == "asc":
query = query.order_by(Feedback.data['rating'].as_string().desc())
elif order_by == 'updated_at':
if direction == 'asc':
query = query.order_by(Feedback.updated_at.asc())
else:
query = query.order_by(Feedback.updated_at.desc())
@@ -270,9 +258,7 @@ class FeedbackTable:
for feedback, user in items:
feedback_model = FeedbackModel.model_validate(feedback)
user_model = UserResponse.model_validate(user)
feedbacks.append(
FeedbackUserResponse(**feedback_model.model_dump(), user=user_model)
)
feedbacks.append(FeedbackUserResponse(**feedback_model.model_dump(), user=user_model))
return FeedbackListResponse(items=feedbacks, total=total)
@@ -280,14 +266,10 @@ class FeedbackTable:
with get_db_context(db) as db:
return [
FeedbackModel.model_validate(feedback)
for feedback in db.query(Feedback)
.order_by(Feedback.updated_at.desc())
.all()
for feedback in db.query(Feedback).order_by(Feedback.updated_at.desc()).all()
]
def get_all_feedback_ids(
self, db: Optional[Session] = None
) -> list[FeedbackIdResponse]:
def get_all_feedback_ids(self, db: Optional[Session] = None) -> list[FeedbackIdResponse]:
with get_db_context(db) as db:
return [
FeedbackIdResponse(
@@ -306,14 +288,11 @@ class FeedbackTable:
.all()
]
def get_feedbacks_for_leaderboard(
self, db: Optional[Session] = None
) -> list[LeaderboardFeedbackData]:
def get_feedbacks_for_leaderboard(self, db: Optional[Session] = None) -> list[LeaderboardFeedbackData]:
"""Fetch only id and data for leaderboard computation (excludes snapshot/meta)."""
with get_db_context(db) as db:
return [
LeaderboardFeedbackData(id=row.id, data=row.data)
for row in db.query(Feedback.id, Feedback.data).all()
LeaderboardFeedbackData(id=row.id, data=row.data) for row in db.query(Feedback.id, Feedback.data).all()
]
def get_model_evaluation_history(
@@ -333,30 +312,26 @@ class FeedbackTable:
rows = db.query(Feedback.created_at, Feedback.data).all()
else:
cutoff = int(time.time()) - (days * 86400)
rows = (
db.query(Feedback.created_at, Feedback.data)
.filter(Feedback.created_at >= cutoff)
.all()
)
rows = db.query(Feedback.created_at, Feedback.data).filter(Feedback.created_at >= cutoff).all()
daily_counts = defaultdict(lambda: {"won": 0, "lost": 0})
daily_counts = defaultdict(lambda: {'won': 0, 'lost': 0})
first_date = None
for created_at, data in rows:
if not data:
continue
if data.get("model_id") != model_id:
if data.get('model_id') != model_id:
continue
rating_str = str(data.get("rating", ""))
if rating_str not in ("1", "-1"):
rating_str = str(data.get('rating', ''))
if rating_str not in ('1', '-1'):
continue
date_str = datetime.fromtimestamp(created_at).strftime("%Y-%m-%d")
if rating_str == "1":
daily_counts[date_str]["won"] += 1
date_str = datetime.fromtimestamp(created_at).strftime('%Y-%m-%d')
if rating_str == '1':
daily_counts[date_str]['won'] += 1
else:
daily_counts[date_str]["lost"] += 1
daily_counts[date_str]['lost'] += 1
# Track first date for this model
if first_date is None or date_str < first_date:
@@ -368,7 +343,7 @@ class FeedbackTable:
if days == 0 and first_date:
# All time: start from first feedback date
start_date = datetime.strptime(first_date, "%Y-%m-%d").date()
start_date = datetime.strptime(first_date, '%Y-%m-%d').date()
num_days = (today - start_date).days + 1
else:
# Fixed range
@@ -377,36 +352,24 @@ class FeedbackTable:
for i in range(num_days):
d = start_date + timedelta(days=i)
date_str = d.strftime("%Y-%m-%d")
counts = daily_counts.get(date_str, {"won": 0, "lost": 0})
result.append(
ModelHistoryEntry(date=date_str, won=counts["won"], lost=counts["lost"])
)
date_str = d.strftime('%Y-%m-%d')
counts = daily_counts.get(date_str, {'won': 0, 'lost': 0})
result.append(ModelHistoryEntry(date=date_str, won=counts['won'], lost=counts['lost']))
return result
def get_feedbacks_by_type(
self, type: str, db: Optional[Session] = None
) -> list[FeedbackModel]:
def get_feedbacks_by_type(self, type: str, db: Optional[Session] = None) -> list[FeedbackModel]:
with get_db_context(db) as db:
return [
FeedbackModel.model_validate(feedback)
for feedback in db.query(Feedback)
.filter_by(type=type)
.order_by(Feedback.updated_at.desc())
.all()
for feedback in db.query(Feedback).filter_by(type=type).order_by(Feedback.updated_at.desc()).all()
]
def get_feedbacks_by_user_id(
self, user_id: str, db: Optional[Session] = None
) -> list[FeedbackModel]:
def get_feedbacks_by_user_id(self, user_id: str, db: Optional[Session] = None) -> list[FeedbackModel]:
with get_db_context(db) as db:
return [
FeedbackModel.model_validate(feedback)
for feedback in db.query(Feedback)
.filter_by(user_id=user_id)
.order_by(Feedback.updated_at.desc())
.all()
for feedback in db.query(Feedback).filter_by(user_id=user_id).order_by(Feedback.updated_at.desc()).all()
]
def update_feedback_by_id(
@@ -462,9 +425,7 @@ class FeedbackTable:
db.commit()
return True
def delete_feedback_by_id_and_user_id(
self, id: str, user_id: str, db: Optional[Session] = None
) -> bool:
def delete_feedback_by_id_and_user_id(self, id: str, user_id: str, db: Optional[Session] = None) -> bool:
with get_db_context(db) as db:
feedback = db.query(Feedback).filter_by(id=id, user_id=user_id).first()
if not feedback:
@@ -473,9 +434,7 @@ class FeedbackTable:
db.commit()
return True
def delete_feedbacks_by_user_id(
self, user_id: str, db: Optional[Session] = None
) -> bool:
def delete_feedbacks_by_user_id(self, user_id: str, db: Optional[Session] = None) -> bool:
with get_db_context(db) as db:
result = db.query(Feedback).filter_by(user_id=user_id).delete()
db.commit()