83 lines
2.4 KiB
Python
83 lines
2.4 KiB
Python
|
"""Reward Calculator — 计算一条审批轨迹的奖励分数。
|
||
|
|
|
||
|
|
诚实分级设计 (对齐兴邦方案):
|
||
|
|
Level 1 (现在能做,不需要用户交互):
|
||
|
|
- 任务完成 +10
|
||
|
|
- 工具调用成功率
|
||
|
|
- 重复调用惩罚
|
||
|
|
- 时间惩罚
|
||
|
|
Level 2 (需前端加 👍/👎 按钮):
|
||
|
|
- 用户显式反馈
|
||
|
|
Level 3 (需人工标注):
|
||
|
|
- 专家打分
|
||
|
|
|
||
|
|
本文件实现 Level 1 + 预留 Level 2/3 参数。
|
||
|
|
不做不诚实的信号: 拒绝用"还款表现"当奖励(周期1-3年无法闭环)。
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
|
||
|
|
def compute_reward_v1(
|
||
|
|
trajectory: dict[str, Any],
|
||
|
|
feedback: str | None = None,
|
||
|
|
expert_score: float | None = None,
|
||
|
|
) -> float:
|
||
|
|
"""计算一条轨迹的奖励(Leve1 l 内置指标 + 可选 Level2 反馈 + Level3 专家分)。
|
||
|
|
|
||
|
|
Args:
|
||
|
|
trajectory: 轨迹 dict(含 steps 和 final_result)
|
||
|
|
feedback: Level2, "positive"/"negative"
|
||
|
|
expert_score: Level3, 1-5 分
|
||
|
|
Returns:
|
||
|
|
float 奖励分
|
||
|
|
"""
|
||
|
|
steps = trajectory.get("steps", [])
|
||
|
|
final = trajectory.get("final_result", {})
|
||
|
|
if not steps:
|
||
|
|
return 0.0
|
||
|
|
|
||
|
|
reward = 0.0
|
||
|
|
|
||
|
|
# 1. 任务完成奖励
|
||
|
|
if final.get("success"):
|
||
|
|
reward += 10.0
|
||
|
|
else:
|
||
|
|
reward -= 10.0
|
||
|
|
|
||
|
|
# 2. 工具调用成功率(成功工具 / 总工具)
|
||
|
|
total_tool_calls = 0
|
||
|
|
tool_success = 0
|
||
|
|
for s in steps:
|
||
|
|
tools = s.get("tools", [])
|
||
|
|
total_tool_calls += len(tools)
|
||
|
|
if tools:
|
||
|
|
tool_success += 1 # 该步骤至少调了工具视为尽力(简化)
|
||
|
|
if total_tool_calls > 0:
|
||
|
|
reward += (tool_success / max(1, total_tool_calls)) * 5.0
|
||
|
|
|
||
|
|
# 3. 步骤数惩罚(越少越好,常规 5 步流水线 5-15 步为佳)
|
||
|
|
n_steps = len(steps)
|
||
|
|
optimal = 5
|
||
|
|
step_penalty = abs(n_steps - optimal) * 0.3
|
||
|
|
reward -= step_penalty
|
||
|
|
|
||
|
|
# 4. 时间惩罚(超过 600 秒 = 10 分钟开始扣)
|
||
|
|
duration_ms = final.get("duration_ms", 0)
|
||
|
|
if duration_ms > 600_000:
|
||
|
|
reward -= ((duration_ms - 600_000) / 60_000) * 1.0
|
||
|
|
|
||
|
|
# ── Level 2: 用户反馈 (可选) ──
|
||
|
|
if feedback == "positive":
|
||
|
|
reward += 20.0
|
||
|
|
elif feedback == "negative":
|
||
|
|
reward -= 10.0
|
||
|
|
|
||
|
|
# ── Level 3: 专家打分 (可选, 1-5 归一化到 -10~+10) ──
|
||
|
|
if expert_score is not None:
|
||
|
|
reward += (expert_score - 3) * 5.0
|
||
|
|
|
||
|
|
reward = max(-30.0, min(reward, 50.0)) # 裁剪
|
||
|
|
return round(reward, 2)
|